Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@miguelbranco80
miguelbranco80 committed Aug 4, 2024
1 parent 87f8fe6 commit e48980b
Show file tree
Hide file tree
Showing 15 changed files with 250 additions and 170 deletions.
129 changes: 116 additions & 13 deletions snapi-client/src/test/scala/raw/compiler/rql2/tests/Rql2CompilerTestContext.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,88 @@ import java.nio.file.{Files, Path, StandardOpenOption}
import scala.collection.mutable
import scala.io.Source

object TestCredentials {

/////////////////////////////////////////////////////////////////////////////
// Dropbox Credentials
/////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////////////////////////////////////////////
// S3 Credentials
/////////////////////////////////////////////////////////////////////////////

val accessKeyId = sys.env("RAW_AWS_ACCESS_KEY_ID")
val secretKeyId = sys.env("RAW_AWS_SECRET_ACCESS_KEY")

// Bucket with public access
val UnitTestPublicBucket = "rawlabs-public-test-data"
val UnitTestPublicBucketCred = S3Credential(None, None, Some("eu-west-1"))

// IAM user 'unit-test-private-bucket', which only has permissions only to access bucket 'rawlabs-private-test-data'
val UnitTestPrivateBucket = "rawlabs-private-test-data"
val UnitTestPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1"))

val UnitTestPrivateBucket2 = "rawlabs-unit-tests"
val UnitTestPrivateBucket2Cred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1"))

val UnitTestEmptyBucketPrivateBucket = "rawlabs-unit-test-empty-bucket"
val UnitTestEmptyBucketPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1"))

val UnitTestListRootPrivateBucket = "rawlabs-unit-test-list-root"
val UnitTestListRootPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1"))

val unitTestPrivateBucketUsEast1 = "rawlabs-unit-tests-us-east-1"
val unitTestPrivateBucketUsEast1Cred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("us-east-1"))

///////////////////////////////////////////////////////////////////////////
// Jdbc Credentials
///////////////////////////////////////////////////////////////////////////

val mysqlTestHost = sys.env("RAW_MYSQL_TEST_HOST")
val mysqlTestDB = sys.env("RAW_MYSQL_TEST_DB")
val mysqlTestUser = sys.env("RAW_MYSQL_TEST_USER")
val mysqlTestPassword = sys.env("RAW_MYSQL_TEST_PASSWORD")
val mysqlCreds = MySqlJdbcLocation(mysqlTestHost, 3306, mysqlTestDB, mysqlTestUser, mysqlTestPassword)
val pgsqlTestHost = sys.env("RAW_PGSQL_TEST_HOST")
val pgsqlTestDB = sys.env("RAW_PGSQL_TEST_DB")
val pgsqlTestUser = sys.env("RAW_PGSQL_TEST_USER")
val pgsqlTestPassword = sys.env("RAW_PGSQL_TEST_PASSWORD")
val pgsqlCreds = PostgresJdbcLocation(pgsqlTestHost, 5432, pgsqlTestDB, pgsqlTestUser, pgsqlTestPassword)
val oracleTestHost = sys.env("RAW_ORACLE_TEST_HOST")
val oracleTestDB = sys.env("RAW_ORACLE_TEST_DB")
val oracleTestUser = sys.env("RAW_ORACLE_TEST_USER")
val oracleTestPassword = sys.env("RAW_ORACLE_TEST_PASSWORD")
val oracleCreds = OracleJdbcLocation(oracleTestHost, 1521, oracleTestDB, oracleTestUser, oracleTestPassword)
val sqlServerTestHost = sys.env("RAW_SQLSERVER_TEST_HOST")
val sqlserverTestDB = sys.env("RAW_SQLSERVER_TEST_DB")
val sqlServerTestUser = sys.env("RAW_SQLSERVER_TEST_USER")
val sqlServerTestPassword = sys.env("RAW_SQLSERVER_TEST_PASSWORD")
val sqlServerCreds = SqlServerJdbcLocation(
sqlServerTestHost,
1433,
sqlserverTestDB,
sqlServerTestUser,
sqlServerTestPassword
)
val teradataTestHost = sys.env("RAW_TERADATA_TEST_HOST")
val teradataTestUser = sys.env("RAW_TERADATA_TEST_USER")
val teradataTestPassword = sys.env("RAW_TERADATA_TEST_PASSWORD")
val teradataCreds = TeraDataJdbcLocation(teradataTestHost, 1025, teradataTestUser, teradataTestPassword)
val snowflakeTestHost = sys.env("RAW_SNOWFLAKE_TEST_HOST")
val snowflakeTestDB = sys.env("RAW_SNOWFLAKE_TEST_DB")
val snowflakeTestUser = sys.env("RAW_SNOWFLAKE_TEST_USER")
val snowflakeTestPassword = sys.env("RAW_SNOWFLAKE_TEST_PASSWORD")
val snowflakeCreds = SnowflakeJdbcLocation(
snowflakeTestHost,
snowflakeTestDB,
snowflakeTestUser,
snowflakeTestPassword,
Map("timezone" -> "UTC")
)
val badMysqlCreds = MySqlJdbcLocation("does-not-exist.raw-labs.com", 3306, "rdbmstest", "t0or", "$up3r$3cr3tValu3")

}

trait Rql2CompilerTestContext
extends RawTestSuite
with Matchers
Expand All @@ -40,6 +122,14 @@ trait Rql2CompilerTestContext
// Simple inferrer
with LocalInferrerTestContext {

private val secrets = new mutable.HashMap[String, String]()

private val s3Credentials = new mutable.HashMap[String, S3Credential]()

private val rdbmsServers = new mutable.HashMap[String, JdbcLocation]()

protected val programOptions = new mutable.HashMap[String, String]()

def authorizedUser: InteractiveUser = InteractiveUser(Uid("janeUid"), "Jane Smith", "jane@example.com")

def runnerScopes: Set[String] = Set.empty
Expand All @@ -48,7 +138,17 @@ trait Rql2CompilerTestContext

def maybeTraceId: Option[String] = None

protected val programOptions = new mutable.HashMap[String, String]()
def secret(name: String, value: String): Unit = {
secrets.put(name, value)
}

def s3Bucket(name: String, bucket: S3Credential): Unit = {
s3Credentials.put(name, bucket)
}

def rdbms(name: String, db: JdbcLocation): Unit = {
rdbmsServers.put(name, db)
}

def option(key: String, value: String): Unit = {
programOptions.put(key, value)
Expand Down Expand Up @@ -468,18 +568,21 @@ trait Rql2CompilerTestContext
maybeArguments: Option[Array[(String, RawValue)]] = None,
scopes: Set[String] = Set.empty,
options: Map[String, String] = Map.empty
): ProgramEnvironment = ProgramEnvironment(
authorizedUser,
maybeArguments,
this.runnerScopes ++ scopes,
Map.empty, // secrets
Map.empty, // Jdbc servers
Map.empty, // http headers
Map.empty, // s3 credentials
this.options ++ options ++ programOptions,
None, // jdbcUrl
maybeTraceId
)
): ProgramEnvironment = {
val user = authorizedUser
ProgramEnvironment(
user,
maybeArguments,
this.runnerScopes ++ scopes,
secrets.toMap,
rdbmsServers.toMap,
Map.empty, // http headers
s3Credentials.toMap,
this.options ++ options ++ programOptions,
None, // jdbcUrl
maybeTraceId
)
}

def parseQuery(code: String): BaseProgram = tryToParse(code) match {
case Right(p) => p
Expand Down
7 changes: 3 additions & 4 deletions snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/AwsPackageTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,14 @@
package raw.compiler.rql2.tests.builtin.credentials

import raw.compiler.rql2.tests.Rql2CompilerTestContext
import raw.creds.api.CredentialsTestContext

trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext {
trait AwsPackageTest extends Rql2CompilerTestContext {

val accessKeyId = sys.env("RAW_AWS_ACCESS_KEY_ID")
val secretAccessKey = sys.env("RAW_AWS_SECRET_ACCESS_KEY")

secret(authorizedUser, "aws-secret-key", secretAccessKey)
secret(authorizedUser, "aws-access-key", accessKeyId)
secret("aws-secret-key", secretAccessKey)
secret("aws-access-key", accessKeyId)

def xmlImplemented: Boolean

Expand Down
5 changes: 2 additions & 3 deletions ...t/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/EnvironmentPackageTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@
package raw.compiler.rql2.tests.builtin.credentials

import raw.compiler.rql2.tests.Rql2CompilerTestContext
import raw.creds.api.CredentialsTestContext

trait EnvironmentPackageTest extends Rql2CompilerTestContext with CredentialsTestContext {
trait EnvironmentPackageTest extends Rql2CompilerTestContext {

secret(authorizedUser, "my-secret", "my-secret-value")
secret("my-secret", "my-secret-value")

test("""Environment.Secret("my-secret")""")(it => it should evaluateTo(""" "my-secret-value" """))

Expand Down
33 changes: 14 additions & 19 deletions ...ient/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/LocationPackageTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,16 @@

package raw.compiler.rql2.tests.builtin.credentials

import raw.creds.dropbox.DropboxTestCreds
import raw.creds.s3.S3TestCreds
import raw.compiler.rql2.tests.Rql2CompilerTestContext
import raw.creds.api.CredentialsTestContext

trait LocationPackageTest
extends Rql2CompilerTestContext
with CredentialsTestContext
with DropboxTestCreds
with S3TestCreds {
trait LocationPackageTest extends Rql2CompilerTestContext {

import raw.compiler.rql2.tests.TestCredentials._

oauth(authorizedUser, "dropbox-refresh-token", dropboxRefreshTokenCredential)

s3Bucket(UnitTestPrivateBucket2, UnitTestPrivateBucket2Cred)

property("raw.sources.dropbox.clientId", dropboxClientId)

test("""
Expand All @@ -37,12 +34,12 @@ trait LocationPackageTest

// reading a public s3 bucket without registering or passing credentials
test(s"""let
| data = Csv.InferAndRead("s3://${UnitTestPublicBucket.name}/students.csv")
| data = Csv.InferAndRead("s3://$UnitTestPublicBucket/students.csv")
|in
| Collection.Count(data)
|""".stripMargin)(it => it should evaluateTo("7"))

test(s"""Location.Ls("s3://${UnitTestPublicBucket.name}/publications/")""") { it =>
test(s"""Location.Ls("s3://$UnitTestPublicBucket/publications/")""") { it =>
it should evaluateTo("""Collection.Build(
| "s3://rawlabs-public-test-data/publications/authors.parquet",
| "s3://rawlabs-public-test-data/publications/authors.hjson",
Expand All @@ -60,7 +57,7 @@ trait LocationPackageTest
|""".stripMargin)
}

test(s"""Location.Ll("s3://${UnitTestPublicBucket.name}/publications/authors.hjson")
test(s"""Location.Ll("s3://$UnitTestPublicBucket/publications/authors.hjson")
|""".stripMargin) { it =>
it should evaluateTo("""Collection.Build(
| Record.Build(
Expand All @@ -78,7 +75,7 @@ trait LocationPackageTest
// Cannot count lists
test(s"""
|let
| data = Location.Ll("s3://${UnitTestPublicBucket.name}/publications")
| data = Location.Ll("s3://$UnitTestPublicBucket/publications")
|in
| List.Count(data)
|""".stripMargin)(it => it should evaluateTo("""12L""".stripMargin))
Expand All @@ -87,20 +84,18 @@ trait LocationPackageTest
test(s"""let
| data = Csv.InferAndRead(
| S3.Build(
| "s3://${UnitTestPrivateBucket.name}/students.csv",
| region = "${UnitTestPrivateBucket.region.get}",
| accessKey = "${UnitTestPrivateBucket.credentials.get.accessKey}",
| secretKey = "${UnitTestPrivateBucket.credentials.get.secretKey}"
| "s3://$UnitTestPrivateBucket/students.csv",
| region = "${UnitTestPrivateBucketCred.region.get}",
| accessKey = "${UnitTestPrivateBucketCred.accessKey.get}",
| secretKey = "${UnitTestPrivateBucketCred.secretKey.get}"
| )
| )
|in
| Collection.Count(data)
|""".stripMargin)(it => it should evaluateTo("7"))

s3Bucket(authorizedUser, UnitTestPrivateBucket2)

// using a private bucket registered in the credentials server
test(s"""String.Read(S3.Build("s3://${UnitTestPrivateBucket2.name}/file1.csv"))
test(s"""String.Read(S3.Build("s3://$UnitTestPrivateBucket2/file1.csv"))
|""".stripMargin)(it => it should evaluateTo(""" "foobar" """))

}
32 changes: 16 additions & 16 deletions ...-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/MySQLPackageTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@

package raw.compiler.rql2.tests.builtin.credentials

import raw.compiler.rql2.tests.Rql2CompilerTestContext
import raw.creds.api.CredentialsTestContext
import raw.creds.jdbc.RDBMSTestCreds
import raw.compiler.rql2.tests.{Rql2CompilerTestContext, TestCredentials}

trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds {
trait MySQLPackageTest extends Rql2CompilerTestContext {

import TestCredentials._

val mysqlRegDb = "registered-db"
val mysqlTable = "tbl1"

rdbms(authorizedUser, mysqlRegDb, mysqlCreds)
rdbms(mysqlRegDb, mysqlCreds)

private val ttt = "\"\"\""
// Trying all types. Not all expressions type as wanted so that
Expand Down Expand Up @@ -112,7 +112,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte

test(
s"""MySQL.InferAndRead("${mysqlCreds.database}", "$mysqlTable",
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}")""".stripMargin
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}")""".stripMargin
) { it =>
it should evaluateTo(
"""[
Expand All @@ -126,7 +126,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
test(
s"""MySQL.Read("${mysqlCreds.database}", "$mysqlTable",
| type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)),
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}")""".stripMargin
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}")""".stripMargin
) { it =>
it should orderEvaluateTo(
"""[
Expand All @@ -142,8 +142,8 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
| d = Location.Describe(Location.Build(
| "mysql://${mysqlCreds.database}/$mysqlTable",
| db_host = "${mysqlCreds.host}",
| db_username = "${mysqlCreds.username.get.toString}",
| db_password = "${mysqlCreds.password.get.toString}"
| db_username = "${mysqlCreds.username}",
| db_password = "${mysqlCreds.password}"
| ))
|in
| d.columns
Expand Down Expand Up @@ -174,7 +174,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
s"""MySQL.Read(
| "${badMysqlCreds.database}", "$mysqlTable",
| type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)),
| host = "${badMysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}"
| host = "${badMysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}"
|)""".stripMargin
)(it => it should runErrorAs(s"""unknown host: ${badMysqlCreds.host}""".stripMargin))

Expand All @@ -184,7 +184,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
s"""MySQL.Read(
| "${mysqlCreds.database}", "$mysqlTable",
| type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)),
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}", port = 1234
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}", port = 1234
|)""".stripMargin
)(it => it should runErrorAs(s"""connect timed out: ${mysqlCreds.database}""".stripMargin))

Expand All @@ -202,7 +202,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
s"""MySQL.Read(
| "${mysqlCreds.database}", "$mysqlTable",
| type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)),
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "wrong!"
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "wrong!"
|)""".stripMargin
)(it => it should runErrorAs("""authentication failed""".stripMargin))

Expand All @@ -218,7 +218,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte

test(
s"""MySQL.InferAndQuery("${mysqlCreds.database}", "SELECT * FROM $mysqlTable",
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}" )""".stripMargin
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}" )""".stripMargin
) { it =>
it should evaluateTo(
"""[
Expand All @@ -244,7 +244,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
test(
s"""MySQL.Query("${mysqlCreds.database}", "SELECT * FROM $mysqlTable",
| type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)),
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}" )""".stripMargin
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}" )""".stripMargin
) { it =>
it should evaluateTo(
"""[
Expand Down Expand Up @@ -297,8 +297,8 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte
| Collection.Count(
| MySQL.Query("${mysqlCreds.database}", "SELECT * FROM " + table,
| type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)),
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get}",
| password = "${mysqlCreds.password.get}")
| host = "${mysqlCreds.host}", username = "${mysqlCreds.username}",
| password = "${mysqlCreds.password}")
| ))""".stripMargin
) { it =>
val error =
Expand Down
Loading

0 comments on commit e48980b

Please sign in to comment.