Introduce a protobuf RAW protocol (#479)

build.sbt

@@ -1,16 +1,9 @@
-import de.heikoseeberger.sbtheader.HeaderPlugin.autoImport._
 import sbt.Keys._
 import sbt._
 
-import java.time.Year
-
 import raw.build.Dependencies._
 import raw.build.BuildSettings._
 
-import java.io.IOException
-import java.nio.file.{Files, Paths}
-import java.nio.charset.StandardCharsets
-
 import scala.sys.process._
 
 import com.jsuereth.sbtpgp.PgpKeys.{publishSigned}
@@ -43,6 +36,7 @@ writeVersionToFile := {
 
 lazy val root = (project in file("."))
   .aggregate(
+    protocol,
     utils,
     sources,
     client,
@@ -108,9 +102,22 @@ lazy val sources = (project in file("sources"))
     )
   )
 
+lazy val protocol = (project in file("protocol"))
+  .enablePlugins(ProtobufPlugin)
+  .settings(
+    commonSettings,
+    commonCompileSettings,
+    testSettings,
+    ProtobufConfig / version := "3.18.0",
+    libraryDependencies += "com.google.protobuf" % "protobuf-java" % (ProtobufConfig / version).value,
+    // Include the protobuf files in the JAR
+    Compile / unmanagedResourceDirectories += (ProtobufConfig / sourceDirectory).value
+  )
+
 lazy val client = (project in file("client"))
   .dependsOn(
-    utils % "compile->compile;test->test"
+    utils % "compile->compile;test->test",
+    protocol % "compile->compile;test->test"
   )
   .settings(
     commonSettings,

client/src/main/java/module-info.java

@@ -21,6 +21,7 @@
   requires com.fasterxml.jackson.datatype.jsr310;
   requires com.fasterxml.jackson.datatype.jdk8;
   requires raw.utils;
+  requires raw.protocol;
 
   exports raw.client.api;
   exports raw.client.writers;

client/src/main/scala/raw/client/api/ProgramEnvironment.scala

@@ -12,99 +12,45 @@
 
 package raw.client.api
 
-import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType}
-import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo}
-import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
+import com.fasterxml.jackson.databind.module.SimpleModule
+import com.fasterxml.jackson.databind.{
+  DeserializationContext,
+  JsonDeserializer,
+  JsonSerializer,
+  ObjectMapper,
+  SerializerProvider
+}
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule}
+
 import raw.utils.RawUid
+import raw.protocol.LocationConfig
 
 final case class ProgramEnvironment(
     uid: RawUid,
     maybeArguments: Option[Array[(String, RawValue)]],
     scopes: Set[String],
     secrets: Map[String, String],
-    jdbcServers: Map[String, JdbcLocation],
-    httpHeaders: Map[String, Map[String, String]],
-    s3Credentials: Map[String, S3Credential],
+    locationConfigs: Map[String, LocationConfig],
     options: Map[String, String],
     jdbcUrl: Option[String] = None,
     maybeTraceId: Option[String] = None
 )
 
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
-@JsonSubTypes(
-  Array(
-    new JsonType(value = classOf[MySqlJdbcLocation], name = "mysql"),
-    new JsonType(value = classOf[OracleJdbcLocation], name = "oracle"),
-    new JsonType(value = classOf[PostgresJdbcLocation], name = "postgres"),
-    new JsonType(value = classOf[SqlServerJdbcLocation], name = "sqlserver"),
-    new JsonType(value = classOf[SnowflakeJdbcLocation], name = "snowflake"),
-    new JsonType(value = classOf[SqliteJdbcLocation], name = "sqlite"),
-    new JsonType(value = classOf[TeradataJdbcLocation], name = "teradata")
-  )
-)
-trait JdbcLocation
-final case class MySqlJdbcLocation(
-    host: String,
-    port: Int,
-    database: String,
-    username: String,
-    password: String
-) extends JdbcLocation
-final case class OracleJdbcLocation(
-    host: String,
-    port: Int,
-    database: String,
-    username: String,
-    password: String
-) extends JdbcLocation
-final case class PostgresJdbcLocation(
-    host: String,
-    port: Int,
-    database: String,
-    username: String,
-    password: String
-) extends JdbcLocation
-final case class SqlServerJdbcLocation(
-    host: String,
-    port: Int,
-    database: String,
-    username: String,
-    password: String
-) extends JdbcLocation
-final case class SnowflakeJdbcLocation(
-    database: String,
-    username: String,
-    password: String,
-    accountIdentifier: String,
-    parameters: Map[String, String]
-) extends JdbcLocation
-final case class SqliteJdbcLocation(
-    path: String
-) extends JdbcLocation
-final case class TeradataJdbcLocation(
-    host: String,
-    port: Int,
-    database: String,
-    username: String,
-    password: String,
-    parameters: Map[String, String]
-) extends JdbcLocation
-
-final case class S3Credential(
-    accessKey: Option[String],
-    secretKey: Option[String],
-    region: Option[String]
-)
-
 object ProgramEnvironment {
 
   private val jsonMapper = new ObjectMapper with ClassTagExtensions {
     registerModule(DefaultScalaModule)
     registerModule(new JavaTimeModule())
     registerModule(new Jdk8Module())
+
+    // Register custom serializer and deserializer for LocationConfig
+    val customModule = new SimpleModule()
+    customModule.addSerializer(classOf[LocationConfig], new LocationConfigSerializer)
+    customModule.addDeserializer(classOf[LocationConfig], new LocationConfigDeserializer)
+    registerModule(customModule)
   }
 
   private val reader = jsonMapper.readerFor[ProgramEnvironment]
@@ -119,3 +65,15 @@ object ProgramEnvironment {
   }
 
 }
+
+class LocationConfigSerializer extends JsonSerializer[LocationConfig] {
+  override def serialize(value: LocationConfig, gen: JsonGenerator, serializers: SerializerProvider): Unit = {
+    gen.writeBinary(value.toByteArray)
+  }
+}
+
+class LocationConfigDeserializer extends JsonDeserializer[LocationConfig] {
+  override def deserialize(p: JsonParser, ctxt: DeserializationContext): LocationConfig = {
+    LocationConfig.parseFrom(p.getBinaryValue)
+  }
+}

project/plugins.sbt

@@ -19,3 +19,5 @@ addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1")
 libraryDependencies += "commons-io" % "commons-io" % "2.11.0"
 
 addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.8.0")
+
+addSbtPlugin("com.github.sbt" % "sbt-protobuf" % "0.8.0")

protocol/src/main/java/module-info.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 RAW Labs S.A.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0, included in the file
+ * licenses/APL.txt.
+ */
+
+module raw.protocol {
+  requires com.google.protobuf;
+
+  exports raw.protocol;
+}

protocol/src/main/java/raw/protocol/Placeholder.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 RAW Labs S.A.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0, included in the file
+ * licenses/APL.txt.
+ */
+
+package raw.protocol;
+
+class Placeholder {
+  // Placeholder class to allow the module-info.java file to be compiled
+}

protocol/src/main/protobuf/raw/protocol/protocol.proto

@@ -0,0 +1,148 @@
+syntax = "proto3";
+
+option java_package = "raw.protocol";
+option java_multiple_files = true;
+
+package raw.protocol;
+
+message LocationConfig {
+    oneof config {
+        S3Config s3 = 1;
+        MySqlConfig mysql = 2;
+        OracleConfig oracle = 3;
+        PostgreSQLConfig postgresql = 4;
+        SQLServerConfig sqlserver = 5;
+        SnowflakeConfig snowflake = 6;
+        SqliteConfig sqlite = 7;
+        TeradataConfig teradata = 8;
+        GitHubConfig github = 9;
+        JiraConfig jira = 10;
+        ConfluenceConfig confluence = 11;
+        SalesforceConfig salesforce = 12;
+        DropboxAccessTokenConfig dropboxAccessToken = 13;
+        DropboxUsernamePasswordConfig dropboxUsernamePassword = 14;
+        HttpHeadersConfig httpHeaders = 15;
+        SecretConfig secret = 99;
+    }
+}
+
+message S3Config {
+    optional S3AccessSecretKey accessSecretKey = 1;
+    optional string region = 2;
+}
+
+message S3AccessSecretKey {
+    string accessKey = 1;
+    string secretKey = 2;
+}
+
+message MySqlConfig {
+    string host = 1;
+    int32 port = 2;
+    string database = 3;
+    string user = 4;
+    string password = 5;
+}
+
+message OracleConfig {
+    string host = 1;
+    int32 port = 2;
+    string database = 3;
+    string user = 4;
+    string password = 5;
+    optional string schema = 6;
+}
+
+message PostgreSQLConfig {
+    string host = 1;
+    int32 port = 2;
+    string database = 3;
+    string user = 4;
+    string password = 5;
+    optional string schema = 6;
+}
+
+message SQLServerConfig {
+    string host = 1;
+    int32 port = 2;
+    string database = 3;
+    string user = 4;
+    string password = 5;
+    optional string schema = 6;
+}
+
+message SnowflakeConfig {
+    string database = 1;
+    string user = 2;
+    string password = 3;
+    string accountIdentifier = 4;
+    map<string, string> parameters = 5;
+    optional string schema = 6;
+}
+
+message SqliteConfig {
+    string path = 1;
+    optional string schema = 2;
+}
+
+message TeradataConfig {
+    string host = 1;
+    int32 port = 2;
+    string database = 3;
+    string user = 4;
+    string password = 5;
+    map<string, string> parameters = 6;
+    optional string schema = 7;
+}
+
+message GitHubConfig {
+    string token = 1;
+    optional string baseUrl = 2;
+}
+
+message JiraConfig {
+    string baseUrl = 1;
+    string username = 2;
+    string token = 3;
+    JiraTokenType tokenType = 4;
+}
+
+enum JiraTokenType {
+    STANDARD_ACCESS_TOKEN = 0;
+    PERSONAL_ACCESS_TOKEN = 1;
+}
+
+message ConfluenceConfig {
+    string baseUrl = 1;
+    string username = 2;
+    string token = 3;
+}
+
+message SalesforceConfig {
+    string url = 1;
+    string username = 2;
+    string password = 3;
+    string securityToken = 4;
+    string clientId = 5;
+    string apiVersion = 6;
+    repeated string customObjects = 7;
+}
+
+message DropboxAccessTokenConfig {
+    string accessToken = 1;
+}
+
+message DropboxUsernamePasswordConfig {
+    string username = 1;
+    string password = 2;
+}
+
+message HttpHeadersConfig {
+    map<string, string> headers = 1;
+}
+
+// This is not used in practice but is kept for compatibility with FDW interface.
+message SecretConfig {
+    string name = 1;
+    string value = 2;
+}