From da9cfe9896e0722e4bb2c463ae587732b4e57194 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 12 Jan 2024 10:50:58 -0500 Subject: [PATCH] feat(go/adbc/driver/flightsql): enable incremental queries Closes #1451. --- CONTRIBUTING.md | 2 +- go/adbc/adbc.go | 4 +- go/adbc/driver/driverbase/database.go | 2 +- go/adbc/driver/driverbase/driver.go | 2 +- .../driver/flightsql/cmd/testserver/main.go | 10 +- .../flightsql/flightsql_adbc_server_test.go | 389 +++++++++++++++++- .../driver/flightsql/flightsql_adbc_test.go | 14 +- .../driver/flightsql/flightsql_connection.go | 28 +- .../driver/flightsql/flightsql_database.go | 6 +- go/adbc/driver/flightsql/flightsql_driver.go | 2 +- .../driver/flightsql/flightsql_statement.go | 178 +++++++- go/adbc/driver/flightsql/record_reader.go | 10 +- .../driver/flightsql/record_reader_test.go | 12 +- go/adbc/driver/internal/shared_utils.go | 6 +- go/adbc/driver/panicdummy/panicdummy_adbc.go | 6 +- go/adbc/driver/snowflake/connection.go | 4 +- go/adbc/driver/snowflake/driver.go | 2 +- go/adbc/driver/snowflake/driver_test.go | 8 +- go/adbc/driver/snowflake/record_reader.go | 10 +- go/adbc/driver/snowflake/statement.go | 6 +- go/adbc/drivermgr/wrapper.go | 6 +- go/adbc/drivermgr/wrapper_sqlite_test.go | 6 +- go/adbc/go.mod | 9 +- go/adbc/go.sum | 14 +- go/adbc/pkg/_tmpl/driver.go.tmpl | 8 +- go/adbc/pkg/flightsql/driver.go | 8 +- go/adbc/pkg/panicdummy/driver.go | 8 +- go/adbc/pkg/snowflake/driver.go | 8 +- go/adbc/sqldriver/driver.go | 10 +- go/adbc/sqldriver/driver_internals_test.go | 10 +- go/adbc/sqldriver/flightsql/flightsql.go | 2 +- go/adbc/sqldriver/flightsql/flightsql_test.go | 8 +- go/adbc/standard_schemas.go | 2 +- go/adbc/utils/utils.go | 2 +- go/adbc/validation/validation.go | 6 +- .../tools/create-go-vendor-archive.R | 2 +- 36 files changed, 671 insertions(+), 139 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c5ccaf219..b7c5249d4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -347,7 +347,7 @@ $ cd go/adbc && go-licenses report ./... \ --ignore github.com/apache/arrow/go/v11 \ --ignore github.com/apache/arrow/go/v12 \ --ignore github.com/apache/arrow/go/v13 \ - --ignore github.com/apache/arrow/go/v14 \ + --ignore github.com/apache/arrow/go/v15 \ --template ../../license.tpl > ../../LICENSE.txt 2> /dev/null ``` diff --git a/go/adbc/adbc.go b/go/adbc/adbc.go index 71a75dafa8..078135c853 100644 --- a/go/adbc/adbc.go +++ b/go/adbc/adbc.go @@ -40,8 +40,8 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) diff --git a/go/adbc/driver/driverbase/database.go b/go/adbc/driver/driverbase/database.go index 7f32510c8e..07d6c12c27 100644 --- a/go/adbc/driver/driverbase/database.go +++ b/go/adbc/driver/driverbase/database.go @@ -21,7 +21,7 @@ import ( "context" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/slog" ) diff --git a/go/adbc/driver/driverbase/driver.go b/go/adbc/driver/driverbase/driver.go index acd182f8a1..de9eaf1576 100644 --- a/go/adbc/driver/driverbase/driver.go +++ b/go/adbc/driver/driverbase/driver.go @@ -22,7 +22,7 @@ package driverbase import ( "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" ) // DriverImpl is an interface that drivers implement to provide diff --git a/go/adbc/driver/flightsql/cmd/testserver/main.go b/go/adbc/driver/flightsql/cmd/testserver/main.go index bb3053d6c6..0ecda6c3b4 100644 --- a/go/adbc/driver/flightsql/cmd/testserver/main.go +++ b/go/adbc/driver/flightsql/cmd/testserver/main.go @@ -31,11 +31,11 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go index dfd1f6cfd7..675b3e2b9f 100644 --- a/go/adbc/driver/flightsql/flightsql_adbc_server_test.go +++ b/go/adbc/driver/flightsql/flightsql_adbc_server_test.go @@ -23,20 +23,23 @@ import ( "context" "errors" "fmt" + "github.com/google/uuid" "net/textproto" "os" + "strconv" "strings" + "sync" "testing" "time" "github.com/apache/arrow-adbc/go/adbc" driver "github.com/apache/arrow-adbc/go/adbc/driver/flightsql" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/golang/protobuf/ptypes/wrappers" "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" @@ -109,6 +112,10 @@ func TestExecuteSchema(t *testing.T) { suite.Run(t, &ExecuteSchemaTests{}) } +func TestIncrementalPoll(t *testing.T) { + suite.Run(t, &IncrementalPollTests{}) +} + func TestTimeout(t *testing.T) { suite.Run(t, &TimeoutTests{}) } @@ -428,6 +435,376 @@ func (ts *ExecuteSchemaTests) TestQuery() { ts.True(expectedSchema.Equal(schema), schema.String()) } +// ---- IncrementalPoll Tests -------------------- + +type IncrementalQuery struct { + query string + nextIndex int +} + +type IncrementalPollTestServer struct { + flightsql.BaseServer + mu sync.Mutex + queries map[string]*IncrementalQuery + testCases map[string]IncrementalPollTestCase +} + +func (srv *IncrementalPollTestServer) PollFlightInfo(ctx context.Context, desc *flight.FlightDescriptor) (*flight.PollInfo, error) { + srv.mu.Lock() + defer srv.mu.Unlock() + + var val wrapperspb.StringValue + var err error + if err = proto.Unmarshal(desc.Cmd, &val); err != nil { + return nil, err + } + queryId := val.Value + progress := int64(0) + if strings.Contains(queryId, ";") { + parts := strings.SplitN(queryId, ";", 2) + queryId = parts[0] + progress, err = strconv.ParseInt(parts[1], 10, 32) + if err != nil { + return nil, err + } + } + + query, ok := srv.queries[queryId] + if !ok { + return nil, status.Errorf(codes.NotFound, "Query ID not found") + } + + testCase, ok := srv.testCases[query.query] + if !ok { + return nil, status.Errorf(codes.Unimplemented, fmt.Sprintf("Invalid case %s", query.query)) + } + + if testCase.differentRetryDescriptor && progress != int64(query.nextIndex) { + return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Used wrong retry descriptor, expected %d but got %d", query.nextIndex, progress)) + } + + return srv.MakePollInfo(&testCase, query, queryId) +} + +func (srv *IncrementalPollTestServer) PollFlightInfoStatement(ctx context.Context, query flightsql.StatementQuery, desc *flight.FlightDescriptor) (*flight.PollInfo, error) { + queryId := uuid.New().String() + + testCase, ok := srv.testCases[query.GetQuery()] + if !ok { + return nil, status.Errorf(codes.Unimplemented, fmt.Sprintf("Invalid case %s", query.GetQuery())) + } + + srv.mu.Lock() + defer srv.mu.Unlock() + + srv.queries[queryId] = &IncrementalQuery{ + query: query.GetQuery(), + nextIndex: 0, + } + + return srv.MakePollInfo(&testCase, srv.queries[queryId], queryId) +} + +func (srv *IncrementalPollTestServer) PollFlightInfoPreparedStatement(ctx context.Context, query flightsql.PreparedStatementQuery, desc *flight.FlightDescriptor) (*flight.PollInfo, error) { + queryId := uuid.New().String() + req := string(query.GetPreparedStatementHandle()) + + testCase, ok := srv.testCases[req] + if !ok { + return nil, status.Errorf(codes.Unimplemented, fmt.Sprintf("Invalid case %s", req)) + } + + srv.mu.Lock() + defer srv.mu.Unlock() + + srv.queries[queryId] = &IncrementalQuery{ + query: req, + nextIndex: 0, + } + + return srv.MakePollInfo(&testCase, srv.queries[queryId], queryId) +} + +func (srv *IncrementalPollTestServer) BeginTransaction(context.Context, flightsql.ActionBeginTransactionRequest) (id []byte, err error) { + return []byte("txn"), nil +} + +func (srv *IncrementalPollTestServer) EndTransaction(context.Context, flightsql.ActionEndTransactionRequest) error { + return nil +} + +func (srv *IncrementalPollTestServer) CreatePreparedStatement(ctx context.Context, req flightsql.ActionCreatePreparedStatementRequest) (res flightsql.ActionCreatePreparedStatementResult, err error) { + return flightsql.ActionCreatePreparedStatementResult{ + Handle: []byte(req.GetQuery()), + DatasetSchema: arrow.NewSchema([]arrow.Field{ + {Name: "ints", Type: arrow.PrimitiveTypes.Int32}, + }, nil), + }, nil +} + +func (srv *IncrementalPollTestServer) ClosePreparedStatement(ctx context.Context, req flightsql.ActionClosePreparedStatementRequest) error { + return nil +} + +func (srv *IncrementalPollTestServer) MakePollInfo(testCase *IncrementalPollTestCase, query *IncrementalQuery, queryId string) (*flight.PollInfo, error) { + schema := flight.SerializeSchema(arrow.NewSchema([]arrow.Field{ + {Name: "ints", Type: arrow.PrimitiveTypes.Int32}, + }, nil), srv.Alloc) + + pb := wrapperspb.StringValue{Value: queryId} + if testCase.differentRetryDescriptor { + pb.Value = queryId + ";" + strconv.Itoa(query.nextIndex+1) + } + descriptor, err := proto.Marshal(&pb) + if err != nil { + return nil, err + } + + numEndpoints := 0 + for i := 0; i <= query.nextIndex; i++ { + if i >= len(testCase.progress) { + break + } + numEndpoints += testCase.progress[i] + } + endpoints := make([]*flight.FlightEndpoint, numEndpoints) + for i := range endpoints { + endpoints[i] = &flight.FlightEndpoint{ + Ticket: &flight.Ticket{ + Ticket: []byte{}, + }, + } + } + + query.nextIndex++ + pollInfo := flight.PollInfo{ + Info: &flight.FlightInfo{ + Schema: schema, + Endpoint: endpoints, + }, + FlightDescriptor: &flight.FlightDescriptor{ + Type: flight.DescriptorCMD, + Cmd: descriptor, + }, + Progress: proto.Float64(float64(query.nextIndex) / float64(len(testCase.progress))), + } + + if query.nextIndex >= len(testCase.progress) { + if testCase.completeLazily { + if query.nextIndex == len(testCase.progress) { + // Make the client poll one more time + } else { + pollInfo.FlightDescriptor = nil + delete(srv.queries, queryId) + } + + } else { + pollInfo.FlightDescriptor = nil + delete(srv.queries, queryId) + } + } + + return &pollInfo, nil +} + +type IncrementalPollTestCase struct { + // on each poll (including the first), this many new endpoints complete + // making 0 progress is allowed, but not recommended (allow clients to 'long poll') + progress []int + + // use a different retry descriptor for each poll + differentRetryDescriptor bool + + // require one extra poll to get completion (i.e. the last poll will have a nil FlightInfo) + completeLazily bool +} + +type IncrementalPollTests struct { + ServerBasedTests + testCases map[string]IncrementalPollTestCase +} + +func (suite *IncrementalPollTests) SetupSuite() { + suite.testCases = map[string]IncrementalPollTestCase{ + "basic": { + progress: []int{1, 1, 1, 1}, + }, + "basic 2": { + progress: []int{2, 3, 4, 5}, + }, + "basic 3": { + progress: []int{2}, + }, + "descriptor changes": { + progress: []int{1, 1, 1, 1}, + differentRetryDescriptor: true, + }, + "lazy": { + progress: []int{1, 1, 1, 1}, + completeLazily: true, + }, + "lazy 2": { + progress: []int{1, 1, 1, 0}, + completeLazily: true, + }, + "no progress": { + progress: []int{0, 1, 1, 1}, + }, + "no progress 2": { + progress: []int{0, 0, 1, 1}, + }, + "no progress 3": { + progress: []int{0, 0, 1, 0}, + }, + } + + srv := IncrementalPollTestServer{ + queries: make(map[string]*IncrementalQuery), + testCases: suite.testCases, + } + suite.NoError(srv.RegisterSqlInfo(flightsql.SqlInfoFlightSqlServerTransaction, int32(flightsql.SqlTransactionTransaction))) + srv.Alloc = memory.DefaultAllocator + suite.DoSetupSuite(&srv, nil, nil) +} + +func (ts *IncrementalPollTests) TestMaxProgress() { + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + opts := stmt.(adbc.GetSetOptions) + + val, err := opts.GetOptionDouble(adbc.OptionKeyMaxProgress) + ts.NoError(err) + ts.Equal(1.0, val) +} + +func (ts *IncrementalPollTests) TestOptionValue() { + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + opts := stmt.(adbc.GetSetOptions) + + val, err := opts.GetOption(adbc.OptionKeyIncremental) + ts.NoError(err) + ts.Equal(adbc.OptionValueDisabled, val) + + ts.NoError(stmt.SetOption(adbc.OptionKeyIncremental, adbc.OptionValueEnabled)) + + val, err = opts.GetOption(adbc.OptionKeyIncremental) + ts.NoError(err) + ts.Equal(adbc.OptionValueEnabled, val) + + var adbcErr adbc.Error + ts.ErrorAs(stmt.SetOption(adbc.OptionKeyIncremental, "foobar"), &adbcErr) + ts.Equal(adbc.StatusInvalidArgument, adbcErr.Code) +} + +func (ts *IncrementalPollTests) RunOneTestCase(ctx context.Context, stmt adbc.Statement, name string, testCase *IncrementalPollTestCase) { + opts := stmt.(adbc.GetSetOptions) + + for idx, progress := range testCase.progress { + if progress == 0 { + // the driver hides this from us + continue + } + + _, partitions, _, err := stmt.ExecutePartitions(ctx) + ts.NoError(err) + + ts.Equal(uint64(progress), partitions.NumPartitions) + + val, err := opts.GetOptionDouble(adbc.OptionKeyProgress) + ts.NoError(err) + ts.Equal(float64(idx+1)/float64(len(testCase.progress)), val) + } + + // Query completed, but we find out by getting no partitions in this call + _, partitions, _, err := stmt.ExecutePartitions(ctx) + ts.NoError(err) + + ts.Equal(uint64(0), partitions.NumPartitions) +} + +func (ts *IncrementalPollTests) TestQuery() { + ctx := context.Background() + for name, testCase := range ts.testCases { + ts.Run(name, func() { + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + + ts.NoError(stmt.SetOption(adbc.OptionKeyIncremental, adbc.OptionValueEnabled)) + + // Run the query multiple times (we should be able to reuse the statement) + for i := 0; i < 2; i++ { + ts.NoError(stmt.SetSqlQuery(name)) + ts.RunOneTestCase(ctx, stmt, name, &testCase) + } + }) + } +} + +func (ts *IncrementalPollTests) TestQueryPrepared() { + ctx := context.Background() + for name, testCase := range ts.testCases { + ts.Run(name, func() { + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + + ts.NoError(stmt.SetOption(adbc.OptionKeyIncremental, adbc.OptionValueEnabled)) + + // Run the query multiple times (we should be able to reuse the statement) + for i := 0; i < 2; i++ { + ts.NoError(stmt.SetSqlQuery(name)) + ts.NoError(stmt.Prepare(ctx)) + ts.RunOneTestCase(ctx, stmt, name, &testCase) + } + }) + } +} + +func (ts *IncrementalPollTests) TestQueryPreparedTransaction() { + ctx := context.Background() + for name, testCase := range ts.testCases { + ts.Run(name, func() { + ts.NoError(ts.cnxn.(adbc.PostInitOptions).SetOption(adbc.OptionKeyAutoCommit, adbc.OptionValueDisabled)) + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + + ts.NoError(stmt.SetOption(adbc.OptionKeyIncremental, adbc.OptionValueEnabled)) + + // Run the query multiple times (we should be able to reuse the statement) + for i := 0; i < 2; i++ { + ts.NoError(stmt.SetSqlQuery(name)) + ts.NoError(stmt.Prepare(ctx)) + ts.RunOneTestCase(ctx, stmt, name, &testCase) + } + }) + } +} + +func (ts *IncrementalPollTests) TestQueryTransaction() { + ctx := context.Background() + for name, testCase := range ts.testCases { + ts.Run(name, func() { + ts.NoError(ts.cnxn.(adbc.PostInitOptions).SetOption(adbc.OptionKeyAutoCommit, adbc.OptionValueDisabled)) + stmt, err := ts.cnxn.NewStatement() + ts.NoError(err) + defer stmt.Close() + + ts.NoError(stmt.SetOption(adbc.OptionKeyIncremental, adbc.OptionValueEnabled)) + + // Run the query multiple times (we should be able to reuse the statement) + for i := 0; i < 2; i++ { + ts.NoError(stmt.SetSqlQuery(name)) + ts.RunOneTestCase(ctx, stmt, name, &testCase) + } + }) + } +} + // ---- Timeout Tests -------------------- type TimeoutTestServer struct { diff --git a/go/adbc/driver/flightsql/flightsql_adbc_test.go b/go/adbc/driver/flightsql/flightsql_adbc_test.go index 2fcc010de3..b0f786e9ea 100644 --- a/go/adbc/driver/flightsql/flightsql_adbc_test.go +++ b/go/adbc/driver/flightsql/flightsql_adbc_test.go @@ -42,12 +42,12 @@ import ( "github.com/apache/arrow-adbc/go/adbc" driver "github.com/apache/arrow-adbc/go/adbc/driver/flightsql" "github.com/apache/arrow-adbc/go/adbc/validation" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/grpc" @@ -260,7 +260,7 @@ func (s *FlightSQLQuirks) GetMetadata(code adbc.InfoCode) interface{} { case adbc.InfoVendorVersion: return "sqlite 3" case adbc.InfoVendorArrowVersion: - return "14.0.2" + return "15.0.0-SNAPSHOT" } return nil diff --git a/go/adbc/driver/flightsql/flightsql_connection.go b/go/adbc/driver/flightsql/flightsql_connection.go index ee24b75ee2..a3f9f4cc83 100644 --- a/go/adbc/driver/flightsql/flightsql_connection.go +++ b/go/adbc/driver/flightsql/flightsql_connection.go @@ -27,12 +27,12 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/internal" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v14/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v15/arrow/ipc" "github.com/bluele/gcache" "google.golang.org/grpc" grpccodes "google.golang.org/grpc/codes" @@ -886,6 +886,22 @@ func (c *cnxn) executeSubstraitUpdate(ctx context.Context, plan flightsql.Substr return c.cl.ExecuteSubstraitUpdate(ctx, plan, opts...) } +func (c *cnxn) poll(ctx context.Context, query string, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if c.txn != nil { + return c.txn.ExecutePoll(ctx, query, retryDescriptor, opts...) + } + + return c.cl.ExecutePoll(ctx, query, retryDescriptor, opts...) +} + +func (c *cnxn) pollSubstrait(ctx context.Context, plan flightsql.SubstraitPlan, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if c.txn != nil { + return c.txn.ExecuteSubstraitPoll(ctx, plan, retryDescriptor, opts...) + } + + return c.cl.ExecuteSubstraitPoll(ctx, plan, retryDescriptor, opts...) +} + func (c *cnxn) prepare(ctx context.Context, query string, opts ...grpc.CallOption) (*flightsql.PreparedStatement, error) { if c.txn != nil { return c.txn.Prepare(ctx, query, opts...) diff --git a/go/adbc/driver/flightsql/flightsql_database.go b/go/adbc/driver/flightsql/flightsql_database.go index f9537f5097..50736488e1 100644 --- a/go/adbc/driver/flightsql/flightsql_database.go +++ b/go/adbc/driver/flightsql/flightsql_database.go @@ -30,9 +30,9 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/driverbase" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" "github.com/bluele/gcache" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/go/adbc/driver/flightsql/flightsql_driver.go b/go/adbc/driver/flightsql/flightsql_driver.go index cc58a9e15b..ffce7c1a9f 100644 --- a/go/adbc/driver/flightsql/flightsql_driver.go +++ b/go/adbc/driver/flightsql/flightsql_driver.go @@ -38,7 +38,7 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/driverbase" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" "golang.org/x/exp/maps" "google.golang.org/grpc/metadata" ) diff --git a/go/adbc/driver/flightsql/flightsql_statement.go b/go/adbc/driver/flightsql/flightsql_statement.go index 9edcf439a5..59674dfef4 100644 --- a/go/adbc/driver/flightsql/flightsql_statement.go +++ b/go/adbc/driver/flightsql/flightsql_statement.go @@ -22,13 +22,14 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/bluele/gcache" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -109,6 +110,19 @@ func (s *sqlOrSubstrait) executeUpdate(ctx context.Context, cnxn *cnxn, opts ... } } +func (s *sqlOrSubstrait) poll(ctx context.Context, cnxn *cnxn, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if s.sqlQuery != "" { + return cnxn.poll(ctx, s.sqlQuery, retryDescriptor, opts...) + } else if s.substraitPlan != nil { + return cnxn.pollSubstrait(ctx, flightsql.SubstraitPlan{Plan: s.substraitPlan, Version: s.substraitVersion}, retryDescriptor, opts...) + } + + return nil, adbc.Error{ + Code: adbc.StatusInvalidState, + Msg: "[Flight SQL] cannot call ExecuteQuery without a query or prepared statement", + } +} + func (s *sqlOrSubstrait) prepare(ctx context.Context, cnxn *cnxn, opts ...grpc.CallOption) (*flightsql.PreparedStatement, error) { if s.sqlQuery != "" { return cnxn.prepare(ctx, s.sqlQuery, opts...) @@ -122,16 +136,25 @@ func (s *sqlOrSubstrait) prepare(ctx context.Context, cnxn *cnxn, opts ...grpc.C } } +type incrementalState struct { + schema *arrow.Schema + previousInfo *flight.FlightInfo + retryDescriptor *flight.FlightDescriptor + complete bool +} + type statement struct { alloc memory.Allocator cnxn *cnxn clientCache gcache.Cache - hdrs metadata.MD - query sqlOrSubstrait - prepared *flightsql.PreparedStatement - queueSize int - timeouts timeoutOption + hdrs metadata.MD + query sqlOrSubstrait + prepared *flightsql.PreparedStatement + queueSize int + timeouts timeoutOption + incrementalState *incrementalState + progress float64 } func (s *statement) closePreparedStatement() error { @@ -140,6 +163,27 @@ func (s *statement) closePreparedStatement() error { return adbcFromFlightStatusWithDetails(err, header, trailer, "ClosePreparedStatement") } +func (s *statement) clearIncrementalQuery() error { + // retryDescriptor != nil ==> query is in progress + if s.incrementalState != nil { + if !s.incrementalState.complete && s.incrementalState.retryDescriptor != nil { + return adbc.Error{ + Code: adbc.StatusInvalidState, + Msg: "[Flight SQL] Cannot disable incremental execution while a query is in progress, finish execution first", + } + } + s.incrementalState = &incrementalState{} + } + return nil +} + +func (s *statement) poll(ctx context.Context, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if s.prepared != nil { + return s.prepared.ExecutePoll(ctx, s.incrementalState.retryDescriptor, opts...) + } + return s.query.poll(ctx, s.cnxn, s.incrementalState.retryDescriptor, opts...) +} + // Close releases any relevant resources associated with this statement // and closes it (particularly if it is a prepared statement). // @@ -173,6 +217,11 @@ func (s *statement) GetOption(key string) (string, error) { return s.timeouts.queryTimeout.String(), nil case OptionTimeoutUpdate: return s.timeouts.updateTimeout.String(), nil + case adbc.OptionKeyIncremental: + if s.incrementalState != nil { + return adbc.OptionValueEnabled, nil + } + return adbc.OptionValueDisabled, nil } if strings.HasPrefix(key, OptionRPCCallHeaderPrefix) { @@ -221,6 +270,10 @@ func (s *statement) GetOptionDouble(key string) (float64, error) { return s.timeouts.queryTimeout.Seconds(), nil case OptionTimeoutUpdate: return s.timeouts.updateTimeout.Seconds(), nil + case adbc.OptionKeyProgress: + return s.progress, nil + case adbc.OptionKeyMaxProgress: + return 1.0, nil } return 0, adbc.Error{ @@ -260,6 +313,24 @@ func (s *statement) SetOption(key string, val string) error { return s.SetOptionInt(key, int64(size)) case OptionStatementSubstraitVersion: s.query.substraitVersion = val + case adbc.OptionKeyIncremental: + switch val { + case adbc.OptionValueEnabled: + if err := s.clearIncrementalQuery(); err != nil { + return err + } + s.incrementalState = &incrementalState{} + case adbc.OptionValueDisabled: + if err := s.clearIncrementalQuery(); err != nil { + return err + } + s.incrementalState = nil + default: + return adbc.Error{ + Msg: fmt.Sprintf("[Flight SQL] Invalid statement option value %s=%s", key, val), + Code: adbc.StatusInvalidArgument, + } + } default: return adbc.Error{ Msg: "[Flight SQL] Unknown statement option '" + key + "'", @@ -318,7 +389,9 @@ func (s *statement) SetSqlQuery(query string) error { } s.prepared = nil } - + if err := s.clearIncrementalQuery(); err != nil { + return err + } s.query.setSqlQuery(query) return nil } @@ -329,6 +402,10 @@ func (s *statement) SetSqlQuery(query string) error { // // This invalidates any prior result sets on this statement. func (s *statement) ExecuteQuery(ctx context.Context) (rdr array.RecordReader, nrec int64, err error) { + if err := s.clearIncrementalQuery(); err != nil { + return nil, -1, err + } + ctx = metadata.NewOutgoingContext(ctx, s.hdrs) var info *flight.FlightInfo var header, trailer metadata.MD @@ -351,6 +428,10 @@ func (s *statement) ExecuteQuery(ctx context.Context) (rdr array.RecordReader, n // ExecuteUpdate executes a statement that does not generate a result // set. It returns the number of rows affected if known, otherwise -1. func (s *statement) ExecuteUpdate(ctx context.Context) (n int64, err error) { + if err := s.clearIncrementalQuery(); err != nil { + return -1, err + } + ctx = metadata.NewOutgoingContext(ctx, s.hdrs) var header, trailer metadata.MD opts := append([]grpc.CallOption{}, grpc.Header(&header), grpc.Trailer(&trailer), s.timeouts) @@ -396,6 +477,9 @@ func (s *statement) SetSubstraitPlan(plan []byte) error { } s.prepared = nil } + if err := s.clearIncrementalQuery(); err != nil { + return err + } s.query.setSubstraitPlan(plan) return nil @@ -485,13 +569,79 @@ func (s *statement) ExecutePartitions(ctx context.Context) (*arrow.Schema, adbc. var ( info *flight.FlightInfo + poll *flight.PollInfo out adbc.Partitions sc *arrow.Schema err error ) var header, trailer metadata.MD - if s.prepared != nil { + if s.incrementalState != nil { + if s.incrementalState.complete { + schema := s.incrementalState.schema + totalRecords := s.incrementalState.previousInfo.TotalRecords + // Reset the statement for reuse + s.incrementalState = &incrementalState{} + s.progress = 0.0 + return schema, adbc.Partitions{}, totalRecords, nil + } + + backoff := 100 * time.Millisecond + for { + // Keep polling until the query completes or we get new partitions + poll, err = s.poll(ctx, grpc.Header(&header), grpc.Trailer(&trailer), s.timeouts) + if err != nil { + break + } + info = poll.GetInfo() + if info == nil { + // The server is misbehaving + // XXX: should we also issue a query cancellation? + s.incrementalState = &incrementalState{} + s.progress = 0.0 + return nil, adbc.Partitions{}, -1, adbc.Error{ + Msg: "[Flight SQL] Server returned a PollInfo with no FlightInfo", + Code: adbc.StatusInternal, + } + } + info = proto.Clone(info).(*flight.FlightInfo) + // We only return the new endpoints each time + if s.incrementalState.previousInfo != nil { + offset := len(s.incrementalState.previousInfo.Endpoint) + if offset >= len(info.Endpoint) { + info.Endpoint = []*flight.FlightEndpoint{} + } else { + info.Endpoint = info.Endpoint[offset:] + } + } + s.incrementalState.previousInfo = poll.GetInfo() + s.incrementalState.retryDescriptor = poll.GetFlightDescriptor() + s.progress = poll.GetProgress() + + if s.incrementalState.retryDescriptor == nil { + // Query is finished + s.incrementalState.complete = true + break + } else if len(info.Endpoint) > 0 { + // Query made progress + break + } + // Back off before next poll + time.Sleep(backoff) + backoff *= 2 + if backoff > 5000*time.Millisecond { + backoff = 5000 * time.Millisecond + } + } + + // Special case: the query completed but there were no new endpoints. We + // return 0 new partitions, and also reset the statement (because + // returning 0 partitions implies completion) + if s.incrementalState.complete && len(info.Endpoint) == 0 { + s.incrementalState = &incrementalState{} + s.progress = 0.0 + } + } else if s.prepared != nil { info, err = s.prepared.Execute(ctx, grpc.Header(&header), grpc.Trailer(&trailer), s.timeouts) } else { info, err = s.query.execute(ctx, s.cnxn, grpc.Header(&header), grpc.Trailer(&trailer), s.timeouts) @@ -508,6 +658,10 @@ func (s *statement) ExecutePartitions(ctx context.Context) (*arrow.Schema, adbc. } } + if s.incrementalState != nil { + s.incrementalState.schema = sc + } + out.NumPartitions = uint64(len(info.Endpoint)) out.PartitionIDs = make([][]byte, out.NumPartitions) for i, e := range info.Endpoint { diff --git a/go/adbc/driver/flightsql/record_reader.go b/go/adbc/driver/flightsql/record_reader.go index 9a10e3708d..c2ae474a41 100644 --- a/go/adbc/driver/flightsql/record_reader.go +++ b/go/adbc/driver/flightsql/record_reader.go @@ -24,11 +24,11 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/utils" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/bluele/gcache" "golang.org/x/sync/errgroup" "google.golang.org/grpc" diff --git a/go/adbc/driver/flightsql/record_reader_test.go b/go/adbc/driver/flightsql/record_reader_test.go index 28c9046275..83104e63fd 100644 --- a/go/adbc/driver/flightsql/record_reader_test.go +++ b/go/adbc/driver/flightsql/record_reader_test.go @@ -24,12 +24,12 @@ import ( "testing" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/bluele/gcache" "github.com/stretchr/testify/suite" "google.golang.org/grpc" diff --git a/go/adbc/driver/internal/shared_utils.go b/go/adbc/driver/internal/shared_utils.go index 3a5791192c..52d74d9ca7 100644 --- a/go/adbc/driver/internal/shared_utils.go +++ b/go/adbc/driver/internal/shared_utils.go @@ -26,9 +26,9 @@ import ( "time" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) type CatalogAndSchema struct { diff --git a/go/adbc/driver/panicdummy/panicdummy_adbc.go b/go/adbc/driver/panicdummy/panicdummy_adbc.go index 95171591d0..a30fc7fee2 100644 --- a/go/adbc/driver/panicdummy/panicdummy_adbc.go +++ b/go/adbc/driver/panicdummy/panicdummy_adbc.go @@ -25,9 +25,9 @@ import ( "os" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" ) func maybePanic(fname string) { diff --git a/go/adbc/driver/snowflake/connection.go b/go/adbc/driver/snowflake/connection.go index e2f98487ec..681d90384b 100644 --- a/go/adbc/driver/snowflake/connection.go +++ b/go/adbc/driver/snowflake/connection.go @@ -30,8 +30,8 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/internal" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" "github.com/snowflakedb/gosnowflake" ) diff --git a/go/adbc/driver/snowflake/driver.go b/go/adbc/driver/snowflake/driver.go index 3b9d72cc7d..c70ce82663 100644 --- a/go/adbc/driver/snowflake/driver.go +++ b/go/adbc/driver/snowflake/driver.go @@ -24,7 +24,7 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/driverbase" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/snowflakedb/gosnowflake" "golang.org/x/exp/maps" ) diff --git a/go/adbc/driver/snowflake/driver_test.go b/go/adbc/driver/snowflake/driver_test.go index a69a0b0455..850fcd5be6 100644 --- a/go/adbc/driver/snowflake/driver_test.go +++ b/go/adbc/driver/snowflake/driver_test.go @@ -37,10 +37,10 @@ import ( "github.com/apache/arrow-adbc/go/adbc/driver/internal" driver "github.com/apache/arrow-adbc/go/adbc/driver/snowflake" "github.com/apache/arrow-adbc/go/adbc/validation" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/google/uuid" "github.com/snowflakedb/gosnowflake" "github.com/stretchr/testify/require" diff --git a/go/adbc/driver/snowflake/record_reader.go b/go/adbc/driver/snowflake/record_reader.go index c77b578ffc..44057d4ee0 100644 --- a/go/adbc/driver/snowflake/record_reader.go +++ b/go/adbc/driver/snowflake/record_reader.go @@ -27,11 +27,11 @@ import ( "time" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/compute" - "github.com/apache/arrow/go/v14/arrow/ipc" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/snowflakedb/gosnowflake" "golang.org/x/sync/errgroup" ) diff --git a/go/adbc/driver/snowflake/statement.go b/go/adbc/driver/snowflake/statement.go index 57ac079d44..edbaa826a4 100644 --- a/go/adbc/driver/snowflake/statement.go +++ b/go/adbc/driver/snowflake/statement.go @@ -25,9 +25,9 @@ import ( "strings" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/snowflakedb/gosnowflake" "golang.org/x/exp/constraints" ) diff --git a/go/adbc/drivermgr/wrapper.go b/go/adbc/drivermgr/wrapper.go index 63fb9ee9f4..7542873c20 100644 --- a/go/adbc/drivermgr/wrapper.go +++ b/go/adbc/drivermgr/wrapper.go @@ -43,9 +43,9 @@ import ( "unsafe" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" ) type option struct { diff --git a/go/adbc/drivermgr/wrapper_sqlite_test.go b/go/adbc/drivermgr/wrapper_sqlite_test.go index af307a08d5..9d8dd9967a 100644 --- a/go/adbc/drivermgr/wrapper_sqlite_test.go +++ b/go/adbc/drivermgr/wrapper_sqlite_test.go @@ -27,9 +27,9 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/drivermgr" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/adbc/go.mod b/go/adbc/go.mod index 2eff02d9ab..8db9d3cbbc 100644 --- a/go/adbc/go.mod +++ b/go/adbc/go.mod @@ -20,7 +20,7 @@ module github.com/apache/arrow-adbc/go/adbc go 1.19 require ( - github.com/apache/arrow/go/v14 v14.0.2 + github.com/apache/arrow/go/v15 v15.0.0-20240119162530-143a7da1038c github.com/bluele/gcache v0.0.2 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.1 @@ -41,8 +41,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect - github.com/andybalholm/brotli v1.1.0 // indirect - github.com/apache/thrift v0.19.0 // indirect + github.com/apache/arrow/go/v14 v14.0.2 // indirect github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect @@ -64,17 +63,13 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect diff --git a/go/adbc/go.sum b/go/adbc/go.sum index e7de21811e..679e46fe23 100644 --- a/go/adbc/go.sum +++ b/go/adbc/go.sum @@ -13,12 +13,10 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcv github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= -github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk= -github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I= +github.com/apache/arrow/go/v15 v15.0.0-20240119162530-143a7da1038c h1:tGuAIZ7IXzDnfeuOcfFUufeVvXlRTWBGyyv2A8E5mnI= +github.com/apache/arrow/go/v15 v15.0.0-20240119162530-143a7da1038c/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= @@ -75,8 +73,6 @@ github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -92,8 +88,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= @@ -106,10 +100,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/go/adbc/pkg/_tmpl/driver.go.tmpl b/go/adbc/pkg/_tmpl/driver.go.tmpl index 901d164ec0..b712d76269 100644 --- a/go/adbc/pkg/_tmpl/driver.go.tmpl +++ b/go/adbc/pkg/_tmpl/driver.go.tmpl @@ -58,10 +58,10 @@ import ( "unsafe" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "golang.org/x/exp/slog" ) diff --git a/go/adbc/pkg/flightsql/driver.go b/go/adbc/pkg/flightsql/driver.go index d57a91b7e7..8699a1bc51 100644 --- a/go/adbc/pkg/flightsql/driver.go +++ b/go/adbc/pkg/flightsql/driver.go @@ -61,10 +61,10 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/flightsql" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "golang.org/x/exp/slog" ) diff --git a/go/adbc/pkg/panicdummy/driver.go b/go/adbc/pkg/panicdummy/driver.go index fbaa5204cd..d855e9155b 100644 --- a/go/adbc/pkg/panicdummy/driver.go +++ b/go/adbc/pkg/panicdummy/driver.go @@ -61,10 +61,10 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/panicdummy" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "golang.org/x/exp/slog" ) diff --git a/go/adbc/pkg/snowflake/driver.go b/go/adbc/pkg/snowflake/driver.go index 6e2d3bac50..c4e38c2794 100644 --- a/go/adbc/pkg/snowflake/driver.go +++ b/go/adbc/pkg/snowflake/driver.go @@ -61,10 +61,10 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/driver/snowflake" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/cdata" - "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/arrow/memory/mallocator" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/cdata" + "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory/mallocator" "golang.org/x/exp/slog" ) diff --git a/go/adbc/sqldriver/driver.go b/go/adbc/sqldriver/driver.go index 775f3f78a8..abc8404111 100644 --- a/go/adbc/sqldriver/driver.go +++ b/go/adbc/sqldriver/driver.go @@ -31,11 +31,11 @@ import ( "unsafe" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" ) func getIsolationlevel(lvl sql.IsolationLevel) adbc.OptionIsolationLevel { diff --git a/go/adbc/sqldriver/driver_internals_test.go b/go/adbc/sqldriver/driver_internals_test.go index 9981a40ddb..c7418b69a1 100644 --- a/go/adbc/sqldriver/driver_internals_test.go +++ b/go/adbc/sqldriver/driver_internals_test.go @@ -26,11 +26,11 @@ import ( "time" "github.com/apache/arrow-adbc/go/adbc" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/decimal128" - "github.com/apache/arrow/go/v14/arrow/decimal256" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/adbc/sqldriver/flightsql/flightsql.go b/go/adbc/sqldriver/flightsql/flightsql.go index 6ad00e9c20..23425fbe6c 100644 --- a/go/adbc/sqldriver/flightsql/flightsql.go +++ b/go/adbc/sqldriver/flightsql/flightsql.go @@ -22,7 +22,7 @@ import ( "github.com/apache/arrow-adbc/go/adbc/driver/flightsql" "github.com/apache/arrow-adbc/go/adbc/sqldriver" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/memory" ) func init() { diff --git a/go/adbc/sqldriver/flightsql/flightsql_test.go b/go/adbc/sqldriver/flightsql/flightsql_test.go index 7d0b86b23a..4c0b7f68b3 100644 --- a/go/adbc/sqldriver/flightsql/flightsql_test.go +++ b/go/adbc/sqldriver/flightsql/flightsql_test.go @@ -23,10 +23,10 @@ import ( "testing" _ "github.com/apache/arrow-adbc/go/adbc/sqldriver/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql" - "github.com/apache/arrow/go/v14/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/suite" "google.golang.org/grpc" ) diff --git a/go/adbc/standard_schemas.go b/go/adbc/standard_schemas.go index 264f76140c..a441ae0f47 100644 --- a/go/adbc/standard_schemas.go +++ b/go/adbc/standard_schemas.go @@ -17,7 +17,7 @@ package adbc -import "github.com/apache/arrow/go/v14/arrow" +import "github.com/apache/arrow/go/v15/arrow" var ( GetInfoSchema = arrow.NewSchema([]arrow.Field{ diff --git a/go/adbc/utils/utils.go b/go/adbc/utils/utils.go index e1c372c32b..4dd69216b3 100644 --- a/go/adbc/utils/utils.go +++ b/go/adbc/utils/utils.go @@ -17,7 +17,7 @@ package utils -import "github.com/apache/arrow/go/v14/arrow" +import "github.com/apache/arrow/go/v15/arrow" func RemoveSchemaMetadata(schema *arrow.Schema) *arrow.Schema { fields := make([]arrow.Field, len(schema.Fields())) diff --git a/go/adbc/validation/validation.go b/go/adbc/validation/validation.go index 192228075b..c67f8a0deb 100644 --- a/go/adbc/validation/validation.go +++ b/go/adbc/validation/validation.go @@ -28,9 +28,9 @@ import ( "github.com/apache/arrow-adbc/go/adbc" "github.com/apache/arrow-adbc/go/adbc/utils" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v15/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/r/adbcsnowflake/tools/create-go-vendor-archive.R b/r/adbcsnowflake/tools/create-go-vendor-archive.R index a9045b0598..aa8ba7959a 100644 --- a/r/adbcsnowflake/tools/create-go-vendor-archive.R +++ b/r/adbcsnowflake/tools/create-go-vendor-archive.R @@ -41,7 +41,7 @@ withr::with_dir("src/go/adbc", { "arrow-go-v14.0.2/go/arrow/cdata/arrow" ) - dst_go_arrow_cdata_dir <- "vendor/github.com/apache/arrow/go/v14/arrow/cdata/" + dst_go_arrow_cdata_dir <- "vendor/github.com/apache/arrow/go/v15/arrow/cdata/" stopifnot(file.copy(src_go_arrow_cdata_arrow_dir, dst_go_arrow_cdata_dir, recursive = TRUE)) })