From 0aeb6bf842cff481f0d783dfe9ee4a5271759ae0 Mon Sep 17 00:00:00 2001 From: Miguel Branco Date: Tue, 6 Aug 2024 15:02:17 +0200 Subject: [PATCH] Removing credentials from source package (#473) --- build.sbt | 49 +- client/src/main/java/module-info.java | 5 +- .../raw/client/api/CompilerService.scala | 2 +- .../raw/client/api/LocationDescription.scala | 96 --- .../raw/client/api/ProgramEnvironment.scala | 105 ++- .../src/main/java/raw/cli/RawLauncher.java | 6 +- .../client/python/PythonCompilerService.scala | 5 +- .../python/TestPythonCompilerService.scala | 37 +- .../client/rql2/api/Rql2CompilerService.scala | 6 +- .../truffle/Rql2TruffleCompilerService.scala | 58 +- .../rql2/tests/Rql2CompilerTestContext.scala | 134 ++- .../rql2/tests/benchmark/BenchmarkTests.scala | 5 +- .../rql2/tests/benchmark/StressTests.scala | 5 +- .../tests/builtin/BinaryPackageTest.scala | 5 +- .../rql2/tests/builtin/BytePackageTest.scala | 5 +- .../rql2/tests/builtin/CsvPackageTest.scala | 55 +- .../rql2/tests/builtin/DatePackageTest.scala | 5 +- .../tests/builtin/DecimalPackageTest.scala | 5 +- .../tests/builtin/DoublePackageTest.scala | 5 +- .../builtin/EnvironmentPackageTest.scala | 7 +- .../rql2/tests/builtin/ErrorPackageTest.scala | 5 +- .../rql2/tests/builtin/FloatPackageTest.scala | 5 +- .../tests/builtin/FunctionPackageTest.scala | 6 +- .../rql2/tests/builtin/HttpPackageTest.scala | 84 +- .../rql2/tests/builtin/IntPackageTest.scala | 5 +- .../tests/builtin/IntervalPackageTest.scala | 5 +- .../rql2/tests/builtin/JsonPackageTest.scala | 21 +- .../tests/builtin/LocationPackageTest.scala | 5 +- .../rql2/tests/builtin/LongPackageTest.scala | 5 +- .../rql2/tests/builtin/MathPackageTest.scala | 5 +- .../tests/builtin/NullablePackageTest.scala | 5 +- .../builtin/NullableTryablePackageTest.scala | 5 +- .../tests/builtin/RecordPackageTest.scala | 5 +- .../rql2/tests/builtin/RegexPackageTest.scala | 5 +- .../rql2/tests/builtin/S3PackageTest.scala | 10 +- .../rql2/tests/builtin/ShortPackageTest.scala | 5 +- .../tests/builtin/StringPackageTest.scala | 12 +- .../tests/builtin/SuccessPackageTest.scala | 5 +- .../rql2/tests/builtin/TimePackageTest.scala | 5 +- .../tests/builtin/TimestampPackageTest.scala | 5 +- .../rql2/tests/builtin/TryPackageTest.scala | 5 +- .../rql2/tests/builtin/TypePackageTest.scala | 5 +- .../rql2/tests/builtin/XmlPackageTest.scala | 11 +- .../collection/CollectionDistinctTest.scala | 5 +- .../collection/CollectionExplodeTest.scala | 5 +- .../collection/CollectionGroupByTest.scala | 5 +- .../collection/CollectionJoinTest.scala | 5 +- .../collection/CollectionMinMaxTest.scala | 5 +- .../collection/CollectionMkStringTest.scala | 5 +- .../collection/CollectionOrderByTest.scala | 5 +- .../collection/CollectionPackageTest.scala | 5 +- .../collection/CollectionRangeTest.scala | 5 +- .../collection/CollectionUnionTest.scala | 5 +- .../builtin/credentials/AwsPackageTest.scala | 26 +- .../credentials/EnvironmentPackageTest.scala | 12 +- .../credentials/LocationPackageTest.scala | 45 +- .../credentials/MySQLPackageTest.scala | 50 +- .../credentials/OraclePackageTest.scala | 42 +- .../credentials/PostgreSQLPackageTest.scala | 50 +- .../builtin/credentials/S3PackageTest.scala | 36 +- ...eTest.scala => SQLServerPackageTest.scala} | 48 +- .../credentials/SnowflakePackageTest.scala | 63 +- .../tests/builtin/list/ListDistinctTest.scala | 5 +- .../tests/builtin/list/ListExplodeTest.scala | 5 +- .../tests/builtin/list/ListGroupByTest.scala | 5 +- .../tests/builtin/list/ListJoinTest.scala | 5 +- .../tests/builtin/list/ListMinMaxTest.scala | 5 +- .../tests/builtin/list/ListMkStringTest.scala | 5 +- .../tests/builtin/list/ListOrderByTest.scala | 5 +- .../tests/builtin/list/ListPackageTest.scala | 5 +- .../tests/builtin/list/ListUnionTest.scala | 5 +- .../hints/SemanticAnalyzerHintsTest.scala | 5 +- .../rql2/tests/lsp/LspAiValidateTest.scala | 5 +- .../rql2/tests/lsp/LspBrokenCodeTest.scala | 5 +- .../tests/lsp/LspCommentsFormatTest.scala | 5 +- .../lsp/LspCompilationMessagesTest.scala | 5 +- .../rql2/tests/lsp/LspDefinitionTest.scala | 5 +- .../tests/lsp/LspDotAutoCompleteTest.scala | 5 +- .../rql2/tests/lsp/LspFormatCodeTest.scala | 5 +- .../rql2/tests/lsp/LspHoverTest.scala | 5 +- .../rql2/tests/lsp/LspRenameTest.scala | 5 +- .../rql2/tests/lsp/LspValidateTest.scala | 5 +- .../tests/lsp/LspWordAutoCompleteTest.scala | 5 +- .../offheap/KryoPackageTest.scala | 6 +- .../rql2/tests/offheap/OffHeapDatasets.scala | 4 +- .../tests/offheap/OffHeapDistinctTest.scala | 5 +- .../tests/offheap/OffHeapEquiJoinTest.scala | 4 +- .../rql2/tests/offheap/OffHeapGroupTest.scala | 4 +- .../rql2/tests/offheap/OffHeapJoinTest.scala | 4 +- .../tests/offheap/OffHeapOrderByTest.scala | 4 +- .../rql2/tests/output/BinaryOutputTest.scala | 5 +- .../rql2/tests/output/CsvOutputTest.scala | 7 +- .../rql2/tests/output/JsonOutputTest.scala | 5 +- .../rql2/tests/output/TextOutputTest.scala | 5 +- .../parser/FrontendSyntaxAnalyzerTest.scala | 5 +- .../rql2/tests/parser/ListSugarTest.scala | 5 +- .../tests/parser/OperatorPrecedenceTest.scala | 5 +- .../rql2/tests/parser/RecordSugarTest.scala | 5 +- .../rql2/tests/regressions/RD10194Test.scala | 22 +- .../rql2/tests/regressions/RD10220Test.scala | 5 +- .../rql2/tests/regressions/RD10723Test.scala | 5 +- .../regressions/RD10767Test.scala | 32 +- .../rql2/tests/regressions/RD10801Test.scala | 5 +- .../rql2/tests/regressions/RD3742Test.scala | 5 +- .../rql2/tests/regressions/RD3784Test.scala | 5 +- .../rql2/tests/regressions/RD4529Test.scala | 5 +- .../rql2/tests/regressions/RD4981Test.scala | 5 +- .../rql2/tests/regressions/RD5238Test.scala | 5 +- .../rql2/tests/regressions/RD5365Test.scala | 5 +- .../rql2/tests/regressions/RD5393Test.scala | 5 +- .../rql2/tests/regressions/RD5412Test.scala | 7 +- .../rql2/tests/regressions/RD5448Test.scala | 5 +- .../rql2/tests/regressions/RD5484Test.scala | 5 +- .../rql2/tests/regressions/RD5488Test.scala | 5 +- .../rql2/tests/regressions/RD5491Test.scala | 5 +- .../rql2/tests/regressions/RD5644Test.scala | 5 +- .../rql2/tests/regressions/RD5679Test.scala | 5 +- .../rql2/tests/regressions/RD5685Test.scala | 5 +- .../rql2/tests/regressions/RD5691Test.scala | 5 +- .../rql2/tests/regressions/RD5697Test.scala | 5 +- .../rql2/tests/regressions/RD5714Test.scala | 5 +- .../rql2/tests/regressions/RD5722Test.scala | 5 +- .../rql2/tests/regressions/RD572Test.scala | 5 +- .../rql2/tests/regressions/RD5775Test.scala | 5 +- .../rql2/tests/regressions/RD5779Test.scala | 5 +- .../rql2/tests/regressions/RD5784Test.scala | 5 +- .../rql2/tests/regressions/RD5785Test.scala | 5 +- .../rql2/tests/regressions/RD5786Test.scala | 5 +- .../rql2/tests/regressions/RD5797Test.scala | 55 -- .../rql2/tests/regressions/RD5851Test.scala | 5 +- .../rql2/tests/regressions/RD5884Test.scala | 5 +- .../rql2/tests/regressions/RD5893Test.scala | 5 +- .../rql2/tests/regressions/RD5914Test.scala | 5 +- .../rql2/tests/regressions/RD5920Test.scala | 5 +- .../rql2/tests/regressions/RD5921Test.scala | 5 +- .../rql2/tests/regressions/RD5925Test.scala | 5 +- .../rql2/tests/regressions/RD5932Test.scala | 6 +- .../rql2/tests/regressions/RD5968Test.scala | 5 +- .../rql2/tests/regressions/RD5971Test.scala | 5 +- .../rql2/tests/regressions/RD5979Test.scala | 5 +- .../rql2/tests/regressions/RD7924Test.scala | 5 +- .../rql2/tests/regressions/RD7974Test.scala | 5 +- .../rql2/tests/regressions/RD8530Test.scala | 5 +- .../rql2/tests/regressions/RD8764Test.scala | 5 +- .../rql2/tests/regressions/RD8935Test.scala | 5 +- .../rql2/tests/regressions/RD8993Test.scala | 5 +- .../rql2/tests/regressions/RD9137Test.scala | 5 +- .../rql2/tests/regressions/RD9228Test.scala | 5 +- .../rql2/tests/regressions/RD9229Test.scala | 5 +- .../rql2/tests/regressions/RD9255Test.scala | 5 +- .../rql2/tests/regressions/RD9359Test.scala | 5 +- .../rql2/tests/regressions/RD9409Test.scala | 5 +- .../rql2/tests/regressions/RD9445Test.scala | 5 +- .../rql2/tests/regressions/RD9479Test.scala | 5 +- .../rql2/tests/regressions/RD9485Test.scala | 5 +- .../rql2/tests/regressions/RD9554Test.scala | 5 +- .../rql2/tests/regressions/RD9616Test.scala | 5 +- .../rql2/tests/regressions/RD9932Test.scala | 5 +- .../regressions/credentials/RD3084Test.scala | 16 +- .../regressions/credentials/RD4445Test.scala | 34 +- .../regressions/credentials/RD5932Test.scala | 10 +- .../tests/spec/BasicStagedCompilerTest.scala | 5 +- .../rql2/tests/spec/BinaryExpAndTest.scala | 5 +- .../rql2/tests/spec/BinaryExpDivTest.scala | 5 +- .../rql2/tests/spec/BinaryExpEqTest.scala | 5 +- .../rql2/tests/spec/BinaryExpGeTest.scala | 5 +- .../rql2/tests/spec/BinaryExpGtTest.scala | 5 +- .../rql2/tests/spec/BinaryExpLeTest.scala | 5 +- .../rql2/tests/spec/BinaryExpLtTest.scala | 5 +- .../rql2/tests/spec/BinaryExpModTest.scala | 5 +- .../rql2/tests/spec/BinaryExpMultTest.scala | 5 +- .../rql2/tests/spec/BinaryExpNeqTest.scala | 5 +- .../rql2/tests/spec/BinaryExpOrTest.scala | 5 +- .../rql2/tests/spec/BinaryExpPlusTest.scala | 5 +- .../rql2/tests/spec/BinaryExpSubTest.scala | 5 +- .../rql2/tests/spec/ClosureTest.scala | 5 +- .../compiler/rql2/tests/spec/ConstTest.scala | 5 +- .../compiler/rql2/tests/spec/ErrorsTest.scala | 5 +- .../compiler/rql2/tests/spec/FunAbsTest.scala | 5 +- .../rql2/tests/spec/IfThenElseTest.scala | 5 +- .../rql2/tests/spec/ImplicitCastTest.scala | 5 +- .../rql2/tests/spec/JoinWithTryRowsTest.scala | 5 +- .../rql2/tests/spec/LetBindTest.scala | 5 +- .../rql2/tests/spec/LetFunRecTest.scala | 5 +- .../compiler/rql2/tests/spec/LetFunTest.scala | 5 +- .../rql2/tests/spec/LetTypeTest.scala | 5 +- .../rql2/tests/spec/MigrationTest.scala | 5 +- .../rql2/tests/spec/PackageNameTest.scala | 5 +- .../compiler/rql2/tests/spec/ProjTest.scala | 5 +- .../rql2/tests/spec/PropagationTest.scala | 5 +- .../rql2/tests/spec/StagedCompilerTest.scala | 5 +- .../rql2/tests/spec/UnaryExpNegTest.scala | 5 +- .../rql2/tests/spec/UnaryExpNotTest.scala | 5 +- .../Rql2TruffleCompilerTestContext.scala | 7 +- ...uffleWithLocalCredentialsTestContext.scala | 23 - .../benchmark/TruffleBenchmarkTest.scala | 21 - .../truffle/builtin/TruffleBuiltInTest.scala | 52 -- .../collection/TruffleCollectionTest.scala | 46 - .../builtin/list/TruffleListTest.scala | 29 - .../rql2/truffle/hints/TruffleHintsTest.scala | 21 - .../rql2/truffle/lsp/TruffleLspTest.scala | 36 - .../truffle/offheap/TruffleOffHeapTest.scala | 23 - .../truffle/output/TruffleOutputTest.scala | 22 - .../truffle/parser/TruffleParserTest.scala | 26 - .../regressions/TruffleRegressionTest.scala | 77 -- .../rql2/truffle/spec/TruffleSpecTest.scala | 57 -- snapi-frontend/src/main/java/module-info.java | 6 +- .../raw/compiler/base/CompilerContext.scala | 6 +- .../raw/compiler/base/NormalizeMap.scala | 47 -- .../raw/compiler/rql2/ImplicitCasts.scala | 3 +- .../scala/raw/compiler/rql2/Propagation.scala | 58 +- .../raw/compiler/rql2/Rql2TypeUtils.scala | 66 ++ .../raw/compiler/rql2/SemanticAnalyzer.scala | 6 +- .../raw/compiler/rql2/StagedCompiler.scala | 78 +- .../rql2/api/EntryExtensionProvider.scala | 2 +- .../rql2/api/LocationDescription.scala | 703 ++++++++++++++++ .../compiler/rql2/api/PackageExtension.scala | 69 +- .../scala/raw/compiler/rql2/api/Values.scala | 8 +- .../compiler/rql2/builtin/CsvPackage.scala | 27 +- .../compiler/rql2/builtin/HttpPackage.scala | 89 +- .../InMemoryLocationValueBuilder.scala | 36 - .../compiler/rql2/builtin/JsonPackage.scala | 31 +- .../rql2/builtin/LocationPackage.scala | 31 +- .../rql2/builtin/LocationPackageBuilder.scala | 9 +- .../compiler/rql2/builtin/MySQLPackage.scala | 117 ++- .../compiler/rql2/builtin/OraclePackage.scala | 119 ++- .../rql2/builtin/PostgreSQLPackage.scala | 119 ++- .../raw/compiler/rql2/builtin/S3Package.scala | 2 +- .../rql2/builtin/SQLServerPackage.scala | 119 ++- .../rql2/builtin/SnowflakePackage.scala | 146 +++- .../builtin/SqlTableExtensionHelper.scala | 116 --- .../compiler/rql2/builtin/StringPackage.scala | 5 +- .../compiler/rql2/builtin/XmlPackage.scala | 6 +- .../raw/compiler/rql2/source/SourceTree.scala | 4 + .../scala/raw/compiler/utils/package.scala | 3 - .../raw/inferrer/api/InferrerProperties.scala | 30 +- .../raw/inferrer/api/InferrerService.scala | 7 +- .../api/InferrerServiceProvider.scala | 7 +- .../inferrer/api/InputFormatDescriptors.scala | 14 +- .../raw/inferrer/local/EncodingInferrer.scala | 5 +- .../inferrer/local/LocalInferrerService.scala | 84 +- .../inferrer/local/auto/AutoInferrer.scala | 40 +- .../raw/inferrer/local/csv/CsvInferrer.scala | 6 +- .../inferrer/local/hjson/HjsonInferrer.scala | 4 +- .../inferrer/local/jdbc/JdbcInferrer.scala | 4 +- .../inferrer/local/json/JsonInferrer.scala | 4 +- .../inferrer/local/text/TextInferrer.scala | 4 +- .../raw/inferrer/local/xml/XmlInferrer.scala | 4 +- .../inferrer/local/LocalInferrerTest.scala | 8 +- .../scala/raw/inferrer/local/RD10260.scala | 2 - .../scala/raw/inferrer/local/RD10439.scala | 8 +- .../scala/raw/inferrer/local/RD3852.scala | 6 +- .../inferrer/local/json/JsonOrTypeTest.scala | 15 +- .../dropbox/DropboxTestContext.scala | 0 .../local/LocalLocationsTestContext.scala | 0 snapi-truffle/src/main/java/module-info.java | 10 +- .../http_extension/TruffleHttpCallEntry.java | 79 +- .../snapi/truffle/builtin/jdbc/Jdbc.java | 1 + .../truffle/builtin/jdbc/WithJdbcArgs.java | 44 - ...va => TruffleLocationFromStringEntry.java} | 20 +- .../TruffleMySQLQueryEntry.java | 51 +- .../TruffleOracleQueryEntry.java | 53 +- .../TrufflePostgreSQLQueryEntry.java | 53 +- .../s3_extension/TruffleS3BuildEntry.java | 40 +- .../TruffleSnowflakeQueryEntry.java | 56 +- .../TruffleSQLServerQueryEntry.java | 50 +- .../truffle/compiler/SnapiTruffleEmitter.java | 8 +- .../java/raw/runtime/truffle/RawContext.java | 121 +-- .../java/raw/runtime/truffle/RawLanguage.java | 13 +- .../raw/runtime/truffle/RawLanguageCache.java | 59 +- .../aws_package/AwsV4SignedRequestNode.java | 81 +- .../binary_package/BinaryReadNode.java | 9 +- .../EnvironmentSecretNode.java | 10 +- .../builtin/http_package/HttpReadNode.java | 16 +- .../builtin}/jdbc/JdbcQueryNode.java | 9 +- .../location_package/LocationBuildNode.java | 143 ---- .../LocationDescribeNode.java | 2 +- .../LocationFromHttpNode.java | 211 +++++ .../LocationFromMySQLCredentialNode.java | 55 ++ .../LocationFromMySQLNode.java | 67 ++ .../LocationFromOracleCredentialNode.java | 55 ++ .../LocationFromOracleNode.java | 67 ++ .../LocationFromPostgreSQLCredentialNode.java | 55 ++ .../LocationFromPostgreSQLNode.java | 67 ++ .../location_package/LocationFromS3Node.java | 99 +++ .../LocationFromSQLServerCredentialNode.java | 55 ++ .../LocationFromSQLServerNode.java | 67 ++ .../LocationFromSnowflakeCredentialNode.java | 55 ++ .../LocationFromSnowflakeNode.java | 107 +++ .../LocationFromStringNode.java | 55 ++ .../location_package/LocationLlNode.java | 14 +- .../location_package/LocationLsNode.java | 12 +- .../string_package/StringFromNode.java | 6 + .../string_package/StringReadLinesNode.java | 10 +- .../string_package/StringReadNode.java | 10 +- .../ast/expressions/literals/IntNode.java | 18 +- .../literals/LocationConstNode.java | 46 + .../truffle/ast/io/jdbc/JdbcQuery.java | 28 +- .../ast/io/json/reader/JsonReadValueNode.java | 6 +- .../io/xml/parser/XmlParseCollectionNode.java | 3 - .../ast/io/xml/parser/XmlReadValueNode.java | 7 +- .../rdbms/JdbcReaderRawTruffleException.java | 4 +- .../collection/StaticInitializers.java | 31 +- .../compute_next/ComputeNextNodes.java | 33 +- .../sources/JdbcQueryComputeNext.java | 15 +- .../iterable/sources/JdbcQueryCollection.java | 10 +- .../runtime/primitives/LocationObject.java | 174 ++-- .../raw/runtime/truffle/utils/IOUtils.java | 10 +- .../truffle/utils/TruffleCharInputStream.java | 2 +- .../truffle/utils/TruffleInputStream.java | 13 +- .../raw.compiler.rql2.api.EntryExtension | 2 +- sources/src/main/java/module-info.java | 19 +- sources/src/main/resources/reference.conf | 46 +- .../scala/raw/auth/api/AuthException.scala | 27 - .../src/main/scala/raw/auth/api/Tokens.scala | 33 - .../scala/raw/creds/api/Credentials.scala | 297 ------- .../raw/creds/api/CredentialsService.scala | 260 ------ .../api/CredentialsServiceProvider.scala | 32 - .../scala/raw/creds/api/OAuth2Provider.scala | 28 - .../raw/creds/client/ClientCredentials.scala | 383 --------- .../client/ClientCredentialsException.scala | 17 - .../client/ClientCredentialsService.scala | 297 ------- .../creds/local/LocalCredentialsService.scala | 158 ---- .../raw/creds/oauth2/api/OAuth2Client.scala | 78 -- .../oauth2/auth0/Auth0OAuth2Client.scala | 131 --- .../oauth2/dropbox/DropboxOAuth2Client.scala | 141 ---- .../oauth2/generic/GenericOAuth2Client.scala | 20 - .../google/GoogleApiKeyOAuth2Client.scala | 134 --- .../linkedin/LinkedInOAuth2Client.scala | 109 --- .../oauth2/shopify/ShopifyOAuth2Client.scala | 21 - .../oauth2/twitter/TwitterOAuth2Client.scala | 99 --- .../creds/oauth2/zoho/ZohoOAuth2Client.scala | 224 ----- .../creds/protocol/ProtocolCredentials.scala | 61 -- .../raw/rest/client/ClientExceptions.scala | 47 -- .../scala/raw/rest/client/RestClient.scala | 795 ------------------ .../raw/rest/common/RawHttpHeaders.scala | 19 - .../scala/raw/rest/common/RestError.scala | 23 - .../raw/rest/common/RestJsonMapper.scala | 32 - .../main/scala/raw/sources/api/Location.scala | 8 +- .../raw/sources/api/LocationBuilder.scala | 23 - .../raw/sources/api/LocationProvider.scala | 27 - .../scala/raw/sources/api/SourceContext.scala | 74 -- .../bytestream/api/ByteStreamLocation.scala | 9 +- .../api/ByteStreamLocationBuilder.scala | 22 - .../api/ByteStreamLocationProvider.scala | 45 - .../bytestream/github/GitHubLocation.scala | 76 ++ .../github/GithubByteStreamLocation.scala | 38 - .../GithubByteStreamLocationBuilder.scala | 68 -- .../http/ApacheRuntimeHttpClient.scala | 266 ------ ...lient.scala => HttpByteStreamClient.scala} | 75 +- .../http/HttpByteStreamException.scala} | 12 +- .../http/HttpByteStreamLocation.scala | 48 +- .../http/HttpByteStreamLocationBuilder.scala | 259 ------ .../bytestream/http/RuntimeHttpClient.scala | 41 - .../InMemoryByteStreamLocationBuilder.scala | 31 - .../InMemoryByteStreamLocation.scala | 26 +- .../api/FileSystemLocationBuilder.scala | 23 - .../api/FileSystemLocationProvider.scala | 35 - ...ropboxPath.scala => BaseDropboxPath.scala} | 26 +- .../dropbox/DropboxAccessTokenPath.scala | 45 + .../dropbox/DropboxFileSystem.scala | 51 +- .../DropboxFileSystemLocationBuilder.scala | 66 -- .../dropbox/DropboxUsernamePasswordPath.scala | 52 ++ .../filesystem/local/LocalFileSystem.scala | 8 +- .../LocalFileSystemLocationBuilder.scala | 32 - .../sources/filesystem/local/LocalPath.scala | 5 +- .../mock/MockFileSystemLocationBuilder.scala | 54 -- .../sources/filesystem/mock/MockPath.scala | 8 +- .../sources/filesystem/s3/S3FileSystem.scala | 74 +- .../s3/S3FileSystemLocationBuilder.scala | 66 -- ...3InputStream.scala => S3InputStream.scala} | 26 +- .../raw/sources/filesystem/s3/S3Path.scala | 40 +- .../raw/sources/jdbc/api/JdbcClient.scala | 30 +- .../jdbc/api/JdbcLocationBuilder.scala | 22 - .../jdbc/api/JdbcLocationProvider.scala | 35 - .../sources/jdbc/api/JdbcSchemaLocation.scala | 7 +- .../jdbc/api/JdbcSchemaLocationBuilder.scala | 22 - .../jdbc/api/JdbcSchemaLocationProvider.scala | 32 - ...ocation.scala => JdbcServerLocation.scala} | 7 +- .../sources/jdbc/api/JdbcTableLocation.scala | 14 +- .../jdbc/api/JdbcTableLocationBuilder.scala | 22 - .../jdbc/api/JdbcTableLocationProvider.scala | 40 - .../raw/sources/jdbc/mysql/MySqlClient.scala | 28 +- .../raw/sources/jdbc/mysql/MySqlClients.scala | 36 - .../jdbc/mysql/MySqlLocationBuilder.scala | 34 - .../raw/sources/jdbc/mysql/MySqlSchema.scala | 36 - .../jdbc/mysql/MySqlSchemaLocation.scala | 50 ++ .../mysql/MySqlSchemaLocationBuilder.scala | 34 - ...cation.scala => MySqlServerLocation.scala} | 19 +- .../raw/sources/jdbc/mysql/MySqlTable.scala | 25 - .../jdbc/mysql/MySqlTableLocation.scala | 39 + .../mysql/MySqlTableLocationBuilder.scala | 34 - .../sources/jdbc/oracle/OracleClient.scala | 44 +- .../sources/jdbc/oracle/OracleClients.scala | 36 - .../jdbc/oracle/OracleLocationBuilder.scala | 34 - .../sources/jdbc/oracle/OracleSchema.scala | 38 - .../jdbc/oracle/OracleSchemaLocation.scala | 57 ++ .../oracle/OracleSchemaLocationBuilder.scala | 32 - ...ation.scala => OracleServerLocation.scala} | 29 +- .../raw/sources/jdbc/oracle/OracleTable.scala | 26 - .../jdbc/oracle/OracleTableLocation.scala | 50 ++ .../oracle/OracleTableLocationBuilder.scala | 34 - .../sources/jdbc/pgsql/PostgresqlClient.scala | 35 +- .../jdbc/pgsql/PostgresqlClients.scala | 36 - .../jdbc/pgsql/PostgresqlLocation.scala | 37 - .../pgsql/PostgresqlLocationBuilder.scala | 34 - .../sources/jdbc/pgsql/PostgresqlSchema.scala | 38 - .../jdbc/pgsql/PostgresqlSchemaLocation.scala | 57 ++ .../PostgresqlSchemaLocationBuilder.scala | 33 - .../jdbc/pgsql/PostgresqlServerLocation.scala | 44 + .../sources/jdbc/pgsql/PostgresqlTable.scala | 26 - .../jdbc/pgsql/PostgresqlTableLocation.scala | 50 ++ .../PostgresqlTableLocationBuilder.scala | 35 - .../jdbc/snowflake/SnowflakeClient.scala | 56 +- .../jdbc/snowflake/SnowflakeClients.scala | 39 - .../jdbc/snowflake/SnowflakeLocation.scala | 36 - .../snowflake/SnowflakeLocationBuilder.scala | 34 - .../jdbc/snowflake/SnowflakeSchema.scala | 38 - .../snowflake/SnowflakeSchemaLocation.scala | 68 ++ .../SnowflakeSchemaLocationBuilder.scala | 32 - .../snowflake/SnowflakeServerLocation.scala | 61 ++ .../jdbc/snowflake/SnowflakeTable.scala | 26 - .../snowflake/SnowflakeTableLocation.scala | 56 ++ .../SnowflakeTableLocationBuilder.scala | 34 - .../sources/jdbc/sqlite/SqliteClient.scala | 48 +- .../sources/jdbc/sqlite/SqliteClients.scala | 29 - .../jdbc/sqlite/SqliteLocationBuilder.scala | 34 - ...chema.scala => SqliteSchemaLocation.scala} | 19 +- .../sqlite/SqliteSchemaLocationBuilder.scala | 34 - ...ation.scala => SqliteServerLocation.scala} | 14 +- ...eTable.scala => SqliteTableLocation.scala} | 14 +- .../sqlite/SqliteTableLocationBuilder.scala | 34 - .../jdbc/sqlserver/SqlServerClient.scala | 28 +- .../jdbc/sqlserver/SqlServerClients.scala | 36 - .../sqlserver/SqlServerLocationBuilder.scala | 34 - .../jdbc/sqlserver/SqlServerSchema.scala | 38 - .../sqlserver/SqlServerSchemaLocation.scala | 57 ++ .../SqlServerSchemaLocationBuilder.scala | 34 - ...on.scala => SqlServerServerLocation.scala} | 29 +- .../jdbc/sqlserver/SqlServerTable.scala | 26 - .../sqlserver/SqlServerTableLocation.scala | 50 ++ .../SqlServerTableLocationBuilder.scala | 34 - .../jdbc/teradata/TeradataClient.scala | 45 +- .../jdbc/teradata/TeradataClients.scala | 27 - .../jdbc/teradata/TeradataLocation.scala | 38 - .../teradata/TeradataLocationBuilder.scala | 34 - .../jdbc/teradata/TeradataSchema.scala | 40 - .../teradata/TeradataSchemaLocation.scala | 65 ++ .../TeradataSchemaLocationBuilder.scala | 33 - .../teradata/TeradataServerLocation.scala | 46 + .../sources/jdbc/teradata/TeradataTable.scala | 26 - .../jdbc/teradata/TeradataTableLocation.scala | 53 ++ .../TeradataTableLocationBuilder.scala | 34 - .../scala/raw/auth/api/AuthTestUsers.scala | 24 - .../creds/api/CredentialsTestContext.scala | 110 --- .../raw/creds/dropbox/DropboxTestCreds.scala | 98 --- .../raw/creds/http/OAuth2TestCreds.scala | 154 ---- .../scala/raw/creds/jdbc/RDBMSTestCreds.scala | 62 -- .../local/LocalCredentialsTestContext.scala | 40 - .../raw/creds/oauth2/TestOauth2Clients.scala | 63 -- .../test/scala/raw/creds/s3/S3TestCreds.scala | 51 -- .../raw/sources/api/SourcesTestContext.scala | 21 - .../http/HttpLocationsTestContext.scala | 2 +- .../bytestream/http/TestHttpServer.scala | 2 +- .../in_memory/TestInMemoryLocation.scala | 45 - .../in_memory/TestInMemoryLocationBuild.scala | 56 -- .../dropbox/TestDropboxFileSystem.scala | 19 +- .../dropbox/TestRootDropboxFileSystem.scala | 13 +- .../local/TestLocalFileSystem.scala | 2 +- .../s3/S3LocationsTestContext.scala | 54 -- .../filesystem/s3/TestLargeDirectory.scala | 15 +- .../filesystem/s3/TestRootS3FileSystem.scala | 39 +- .../filesystem/s3/TestS3FileSystem.scala | 38 +- .../raw/client/sql/SqlCompilerService.scala | 2 +- .../sql/TestSqlCompilerServiceAirports.scala | 12 +- .../sql/TestSqlConnectionFailures.scala | 44 +- .../scala/raw/utils/AuthenticatedUser.scala | 73 -- .../raw/utils/{Uid.scala => RawUid.scala} | 2 +- 478 files changed, 5835 insertions(+), 10546 deletions(-) delete mode 100644 client/src/main/scala/raw/client/api/LocationDescription.scala rename snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/{SqlServerPackageTest.scala => SQLServerPackageTest.scala} (86%) rename snapi-client/src/test/scala/raw/compiler/rql2/{truffle => tests}/offheap/KryoPackageTest.scala (93%) rename snapi-client/src/test/scala/raw/compiler/rql2/{truffle => tests}/regressions/RD10767Test.scala (73%) delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5797Test.scala rename sources/src/main/scala/raw/creds/api/CredentialsException.scala => snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerTestContext.scala (65%) delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleWithLocalCredentialsTestContext.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/benchmark/TruffleBenchmarkTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/TruffleBuiltInTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/collection/TruffleCollectionTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/list/TruffleListTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/hints/TruffleHintsTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/lsp/TruffleLspTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/TruffleOffHeapTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/output/TruffleOutputTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/parser/TruffleParserTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/TruffleRegressionTest.scala delete mode 100644 snapi-client/src/test/scala/raw/compiler/rql2/truffle/spec/TruffleSpecTest.scala delete mode 100644 snapi-frontend/src/main/scala/raw/compiler/base/NormalizeMap.scala create mode 100644 snapi-frontend/src/main/scala/raw/compiler/rql2/api/LocationDescription.scala delete mode 100644 snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/InMemoryLocationValueBuilder.scala delete mode 100644 snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SqlTableExtensionHelper.scala rename {sources => snapi-frontend}/src/test/scala/raw/sources/filesystem/dropbox/DropboxTestContext.scala (100%) rename {sources => snapi-frontend}/src/test/scala/raw/sources/filesystem/local/LocalLocationsTestContext.scala (100%) delete mode 100644 snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/WithJdbcArgs.java rename snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/{TruffleLocationBuildEntry.java => TruffleLocationFromStringEntry.java} (62%) rename snapi-truffle/src/main/java/raw/runtime/truffle/ast/{io => expressions/builtin}/jdbc/JdbcQueryNode.java (86%) delete mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationBuildNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLCredentialNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleCredentialNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLCredentialNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerCredentialNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeCredentialNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromStringNode.java create mode 100644 snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/LocationConstNode.java delete mode 100644 sources/src/main/scala/raw/auth/api/AuthException.scala delete mode 100644 sources/src/main/scala/raw/auth/api/Tokens.scala delete mode 100644 sources/src/main/scala/raw/creds/api/Credentials.scala delete mode 100644 sources/src/main/scala/raw/creds/api/CredentialsService.scala delete mode 100644 sources/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala delete mode 100644 sources/src/main/scala/raw/creds/api/OAuth2Provider.scala delete mode 100644 sources/src/main/scala/raw/creds/client/ClientCredentials.scala delete mode 100644 sources/src/main/scala/raw/creds/client/ClientCredentialsException.scala delete mode 100644 sources/src/main/scala/raw/creds/client/ClientCredentialsService.scala delete mode 100644 sources/src/main/scala/raw/creds/local/LocalCredentialsService.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/api/OAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/auth0/Auth0OAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/dropbox/DropboxOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/generic/GenericOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/google/GoogleApiKeyOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/linkedin/LinkedInOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/shopify/ShopifyOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/twitter/TwitterOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/oauth2/zoho/ZohoOAuth2Client.scala delete mode 100644 sources/src/main/scala/raw/creds/protocol/ProtocolCredentials.scala delete mode 100644 sources/src/main/scala/raw/rest/client/ClientExceptions.scala delete mode 100644 sources/src/main/scala/raw/rest/client/RestClient.scala delete mode 100644 sources/src/main/scala/raw/rest/common/RawHttpHeaders.scala delete mode 100644 sources/src/main/scala/raw/rest/common/RestError.scala delete mode 100644 sources/src/main/scala/raw/rest/common/RestJsonMapper.scala delete mode 100644 sources/src/main/scala/raw/sources/api/LocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/api/LocationProvider.scala delete mode 100644 sources/src/main/scala/raw/sources/api/SourceContext.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationProvider.scala create mode 100644 sources/src/main/scala/raw/sources/bytestream/github/GitHubLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/http/ApacheRuntimeHttpClient.scala rename sources/src/main/scala/raw/sources/bytestream/http/{JavaRuntimeHttpClient.scala => HttpByteStreamClient.scala} (70%) rename sources/src/main/scala/raw/{creds/oauth2/api/RenewedAccessToken.scala => sources/bytestream/http/HttpByteStreamException.scala} (61%) delete mode 100644 sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/http/RuntimeHttpClient.scala delete mode 100644 sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocationBuilder.scala rename sources/src/main/scala/raw/sources/bytestream/{in_memory => inmemory}/InMemoryByteStreamLocation.scala (57%) delete mode 100644 sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationProvider.scala rename sources/src/main/scala/raw/sources/filesystem/dropbox/{DropboxPath.scala => BaseDropboxPath.scala} (62%) create mode 100644 sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxAccessTokenPath.scala delete mode 100644 sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystemLocationBuilder.scala create mode 100644 sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxUsernamePasswordPath.scala delete mode 100644 sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystemLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/filesystem/mock/MockFileSystemLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystemLocationBuilder.scala rename sources/src/main/scala/raw/sources/filesystem/s3/{RawS3InputStream.scala => S3InputStream.scala} (51%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationProvider.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationProvider.scala rename sources/src/main/scala/raw/sources/jdbc/api/{JdbcLocation.scala => JdbcServerLocation.scala} (72%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationProvider.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocationBuilder.scala rename sources/src/main/scala/raw/sources/jdbc/mysql/{MySqlLocation.scala => MySqlServerLocation.scala} (58%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocationBuilder.scala rename sources/src/main/scala/raw/sources/jdbc/oracle/{OracleLocation.scala => OracleServerLocation.scala} (50%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocationBuilder.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlServerLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocationBuilder.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeServerLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocationBuilder.scala rename sources/src/main/scala/raw/sources/jdbc/sqlite/{SqliteSchema.scala => SqliteSchemaLocation.scala} (57%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocationBuilder.scala rename sources/src/main/scala/raw/sources/jdbc/sqlite/{SqliteLocation.scala => SqliteServerLocation.scala} (66%) rename sources/src/main/scala/raw/sources/jdbc/sqlite/{SqliteTable.scala => SqliteTableLocation.scala} (62%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocationBuilder.scala rename sources/src/main/scala/raw/sources/jdbc/sqlserver/{SqlServerLocation.scala => SqlServerServerLocation.scala} (50%) delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClients.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocationBuilder.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchema.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocationBuilder.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataServerLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTable.scala create mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocation.scala delete mode 100644 sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocationBuilder.scala delete mode 100644 sources/src/test/scala/raw/auth/api/AuthTestUsers.scala delete mode 100644 sources/src/test/scala/raw/creds/api/CredentialsTestContext.scala delete mode 100644 sources/src/test/scala/raw/creds/dropbox/DropboxTestCreds.scala delete mode 100644 sources/src/test/scala/raw/creds/http/OAuth2TestCreds.scala delete mode 100644 sources/src/test/scala/raw/creds/jdbc/RDBMSTestCreds.scala delete mode 100644 sources/src/test/scala/raw/creds/local/LocalCredentialsTestContext.scala delete mode 100644 sources/src/test/scala/raw/creds/oauth2/TestOauth2Clients.scala delete mode 100644 sources/src/test/scala/raw/creds/s3/S3TestCreds.scala delete mode 100644 sources/src/test/scala/raw/sources/api/SourcesTestContext.scala delete mode 100644 sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocation.scala delete mode 100644 sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocationBuild.scala delete mode 100644 sources/src/test/scala/raw/sources/filesystem/s3/S3LocationsTestContext.scala delete mode 100644 utils/src/main/scala/raw/utils/AuthenticatedUser.scala rename utils/src/main/scala/raw/utils/{Uid.scala => RawUid.scala} (89%) diff --git a/build.sbt b/build.sbt index 0bf056447..4bb1ec5ed 100644 --- a/build.sbt +++ b/build.sbt @@ -24,7 +24,7 @@ ThisBuild / credentials += Credentials( "raw-labs", sys.env.getOrElse("GITHUB_TOKEN", "") ) -ThisBuild/ resolvers += "Github RAW main repo" at "https://maven.pkg.github.com/raw-labs/raw" +ThisBuild / resolvers += "Github RAW main repo" at "https://maven.pkg.github.com/raw-labs/raw" ThisBuild / javaHome := { val javaHomePath = sys.env.getOrElse("JAVA_HOME", sys.props("java.home")) @@ -83,20 +83,9 @@ lazy val utils = (project in file("utils")) jacksonDeps ) -lazy val client = (project in file("client")) - .dependsOn( - utils % "compile->compile;test->test" - ) - .settings( - commonSettings, - scalaCompileSettings, - testSettings, - libraryDependencies += trufflePolyglot - ) - lazy val sources = (project in file("sources")) .dependsOn( - client % "compile->compile;test->test" + utils % "compile->compile;test->test" ) .settings( commonSettings, @@ -119,6 +108,17 @@ lazy val sources = (project in file("sources")) ) ) +lazy val client = (project in file("client")) + .dependsOn( + utils % "compile->compile;test->test" + ) + .settings( + commonSettings, + scalaCompileSettings, + testSettings, + libraryDependencies += trufflePolyglot + ) + lazy val snapiParser = (project in file("snapi-parser")) .enablePlugins(GenParserPlugin) .settings( @@ -126,16 +126,18 @@ lazy val snapiParser = (project in file("snapi-parser")) commonCompileSettings, javaSrcBasePath := s"${baseDirectory.value}/src/main/java", parserDefinitions := List( - (s"${javaSrcBasePath.value}/raw/compiler/rql2/generated", + ( + s"${javaSrcBasePath.value}/raw/compiler/rql2/generated", "raw.compiler.rql2.generated", s"${javaSrcBasePath.value}/raw/snapi/grammar", - "Snapi") + "Snapi" + ) ), Compile / doc := { file("/dev/null") }, compileOrder := CompileOrder.JavaThenScala, libraryDependencies ++= Seq( antlr4Runtime - ), + ) ) lazy val snapiFrontend = (project in file("snapi-frontend")) @@ -249,7 +251,6 @@ lazy val snapiClient = (project in file("snapi-client")) testSettings ) - lazy val sqlParser = (project in file("sql-parser")) .enablePlugins(GenParserPlugin) .settings( @@ -257,22 +258,24 @@ lazy val sqlParser = (project in file("sql-parser")) commonCompileSettings, javaSrcBasePath := s"${baseDirectory.value}/src/main/java", parserDefinitions := List( - (s"${javaSrcBasePath.value}/raw/client/sql/generated", - "raw.client.sql.generated", - s"${javaSrcBasePath.value}/raw/psql/grammar", - "Psql") + ( + s"${javaSrcBasePath.value}/raw/client/sql/generated", + "raw.client.sql.generated", + s"${javaSrcBasePath.value}/raw/psql/grammar", + "Psql" + ) ), Compile / doc := { file("/dev/null") }, compileOrder := CompileOrder.JavaThenScala, libraryDependencies ++= Seq( - antlr4Runtime, + antlr4Runtime ) ) lazy val sqlClient = (project in file("sql-client")) .dependsOn( client % "compile->compile;test->test", - sqlParser % "compile->compile;test->test", + sqlParser % "compile->compile;test->test" ) .settings( commonSettings, diff --git a/client/src/main/java/module-info.java b/client/src/main/java/module-info.java index 476823afe..bb8912bd5 100644 --- a/client/src/main/java/module-info.java +++ b/client/src/main/java/module-info.java @@ -14,9 +14,12 @@ requires scala.library; requires org.slf4j; requires org.graalvm.polyglot; + requires com.fasterxml.jackson.core; requires com.fasterxml.jackson.databind; requires com.fasterxml.jackson.dataformat.csv; - requires com.fasterxml.jackson.core; + requires com.fasterxml.jackson.scala; + requires com.fasterxml.jackson.datatype.jsr310; + requires com.fasterxml.jackson.datatype.jdk8; requires raw.utils; exports raw.client.api; diff --git a/client/src/main/scala/raw/client/api/CompilerService.scala b/client/src/main/scala/raw/client/api/CompilerService.scala index 8d470191b..9956d7f3d 100644 --- a/client/src/main/scala/raw/client/api/CompilerService.scala +++ b/client/src/main/scala/raw/client/api/CompilerService.scala @@ -81,7 +81,7 @@ object CompilerService { "Arguments" -> environment.maybeArguments .map(args => args.map { case (k, v) => s"$k -> $v" }.mkString("\n")) .getOrElse(""), - "User" -> environment.user.toString, + "Uid" -> environment.uid.toString, "Scopes" -> environment.scopes.mkString(","), "Options" -> environment.options.map { case (k, v) => s"$k -> $v" }.mkString("\n") //"Settings" -> runtimeContext.settings.toString diff --git a/client/src/main/scala/raw/client/api/LocationDescription.scala b/client/src/main/scala/raw/client/api/LocationDescription.scala deleted file mode 100644 index 9e41890c3..000000000 --- a/client/src/main/scala/raw/client/api/LocationDescription.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.client.api - -import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType} -import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} - -import java.time.Duration - -final case class LocationSettingKey(key: String) - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[LocationIntSetting], name = "int"), - new JsonType(value = classOf[LocationStringSetting], name = "string"), - new JsonType(value = classOf[LocationBinarySetting], name = "binary"), - new JsonType(value = classOf[LocationBooleanSetting], name = "boolean"), - new JsonType(value = classOf[LocationDurationSetting], name = "duration"), - new JsonType(value = classOf[LocationKVSetting], name = "kv"), - new JsonType(value = classOf[LocationIntArraySetting], name = "array-int") - ) -) sealed trait LocationSettingValue -final case class LocationIntSetting(value: Int) extends LocationSettingValue -final case class LocationStringSetting(value: String) extends LocationSettingValue -final case class LocationBinarySetting(value: Seq[Byte]) extends LocationSettingValue -final case class LocationBooleanSetting(value: Boolean) extends LocationSettingValue -final case class LocationDurationSetting(value: Duration) extends LocationSettingValue -final case class LocationKVSetting(map: Seq[(String, String)]) extends LocationSettingValue -final case class LocationIntArraySetting(value: Array[Int]) extends LocationSettingValue - -final case class LocationDescription( - url: String, - settings: Map[LocationSettingKey, LocationSettingValue] = Map.empty -) { - - def getIntSetting(key: String): Option[Int] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationIntSetting(value)) => Some(value) - case _ => None - } - } - - def getStringSetting(key: String): Option[String] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationStringSetting(value)) => Some(value) - case _ => None - } - } - - def getBinarySetting(key: String): Option[Array[Byte]] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationBinarySetting(value)) => Some(value.toArray) - case _ => None - } - } - - def getBooleanSetting(key: String): Option[Boolean] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationBooleanSetting(value)) => Some(value) - case _ => None - } - } - - def getDurationSettings(key: String): Option[Duration] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationDurationSetting(value)) => Some(value) - case _ => None - } - } - - def getKVSetting(key: String): Option[Array[(String, String)]] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationKVSetting(value)) => Some(value.toArray) - case _ => None - } - } - - def getIntArraySetting(key: String): Option[Array[Int]] = { - settings.get(LocationSettingKey(key)) match { - case Some(LocationIntArraySetting(value)) => Some(value) - case _ => None - } - } - -} diff --git a/client/src/main/scala/raw/client/api/ProgramEnvironment.scala b/client/src/main/scala/raw/client/api/ProgramEnvironment.scala index c71fa72c4..4b9bd9ff2 100644 --- a/client/src/main/scala/raw/client/api/ProgramEnvironment.scala +++ b/client/src/main/scala/raw/client/api/ProgramEnvironment.scala @@ -12,13 +12,110 @@ package raw.client.api -import raw.utils.AuthenticatedUser +import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType} +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} +import raw.utils.RawUid final case class ProgramEnvironment( - user: AuthenticatedUser, + uid: RawUid, maybeArguments: Option[Array[(String, RawValue)]], scopes: Set[String], + secrets: Map[String, String], + jdbcServers: Map[String, JdbcLocation], + httpHeaders: Map[String, Map[String, String]], + s3Credentials: Map[String, S3Credential], options: Map[String, String], - maybeTraceId: Option[String] = None, - jdbcUrl: Option[String] = None + jdbcUrl: Option[String] = None, + maybeTraceId: Option[String] = None ) + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes( + Array( + new JsonType(value = classOf[MySqlJdbcLocation], name = "mysql"), + new JsonType(value = classOf[OracleJdbcLocation], name = "oracle"), + new JsonType(value = classOf[PostgresJdbcLocation], name = "postgres"), + new JsonType(value = classOf[SqlServerJdbcLocation], name = "sqlserver"), + new JsonType(value = classOf[SnowflakeJdbcLocation], name = "snowflake"), + new JsonType(value = classOf[SqliteJdbcLocation], name = "sqlite"), + new JsonType(value = classOf[TeradataJdbcLocation], name = "teradata") + ) +) +trait JdbcLocation +final case class MySqlJdbcLocation( + host: String, + port: Int, + database: String, + username: String, + password: String +) extends JdbcLocation +final case class OracleJdbcLocation( + host: String, + port: Int, + database: String, + username: String, + password: String +) extends JdbcLocation +final case class PostgresJdbcLocation( + host: String, + port: Int, + database: String, + username: String, + password: String +) extends JdbcLocation +final case class SqlServerJdbcLocation( + host: String, + port: Int, + database: String, + username: String, + password: String +) extends JdbcLocation +final case class SnowflakeJdbcLocation( + database: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String] +) extends JdbcLocation +final case class SqliteJdbcLocation( + path: String +) extends JdbcLocation +final case class TeradataJdbcLocation( + host: String, + port: Int, + database: String, + username: String, + password: String, + parameters: Map[String, String] +) extends JdbcLocation + +final case class S3Credential( + accessKey: Option[String], + secretKey: Option[String], + region: Option[String] +) + +object ProgramEnvironment { + + private val jsonMapper = new ObjectMapper with ClassTagExtensions { + registerModule(DefaultScalaModule) + registerModule(new JavaTimeModule()) + registerModule(new Jdk8Module()) + } + + private val reader = jsonMapper.readerFor[ProgramEnvironment] + private val writer = jsonMapper.writerFor[ProgramEnvironment] + + def serializeToString(env: ProgramEnvironment): String = { + writer.writeValueAsString(env) + } + + def deserializeFromString(str: String): ProgramEnvironment = { + reader.readValue(str) + } + +} diff --git a/launcher/src/main/java/raw/cli/RawLauncher.java b/launcher/src/main/java/raw/cli/RawLauncher.java index c74d0c1bd..d25721036 100644 --- a/launcher/src/main/java/raw/cli/RawLauncher.java +++ b/launcher/src/main/java/raw/cli/RawLauncher.java @@ -16,8 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import raw.client.api.*; -import raw.utils.AuthenticatedUser; -import raw.utils.InteractiveUser; +import raw.utils.RawUid; import raw.utils.RawException; import raw.utils.RawSettings; import scala.Option; @@ -44,7 +43,6 @@ public RawLauncher(String language, PrintWriter writer) { RawSettings rawSettings = new RawSettings(ConfigFactory.load(), ConfigFactory.empty()); this.compilerService = CompilerServiceProvider.apply(language, rawSettings); - AuthenticatedUser user = new InteractiveUser("uid", "name", "email", (Seq) Seq$.MODULE$.empty()); HashMap javaOptions = new HashMap(); javaOptions.put("output-format", "json"); @@ -54,7 +52,7 @@ public RawLauncher(String language, PrintWriter writer) { .asScala() .toMap(scala.Predef.>conforms()); - this.env = new ProgramEnvironment(user, Option.empty(), (Set) Set$.MODULE$.empty(), scalaOptions, Option.empty()); + this.env = new ProgramEnvironment(RawUid("uid"), Option.empty(), (Set) Set$.MODULE$.empty(), scalaOptions, Option.empty()); } diff --git a/python-client/src/main/scala/raw/client/python/PythonCompilerService.scala b/python-client/src/main/scala/raw/client/python/PythonCompilerService.scala index 1dab8f167..0d963f857 100644 --- a/python-client/src/main/scala/raw/client/python/PythonCompilerService.scala +++ b/python-client/src/main/scala/raw/client/python/PythonCompilerService.scala @@ -270,10 +270,7 @@ class PythonCompilerService(engineDefinition: (Engine, Boolean))(implicit protec val ctxBuilder = Context .newBuilder("python") .engine(engine) - .environment("RAW_SETTINGS", settings.renderAsString) - .environment("RAW_USER", environment.user.uid.toString) - .environment("RAW_TRACE_ID", environment.user.uid.toString) - .environment("RAW_SCOPES", environment.scopes.mkString(",")) + .environment("RAW_PROGRAM_ENVIRONMENT", ProgramEnvironment.serializeToString(environment)) .allowExperimentalOptions(true) .allowPolyglotAccess(PolyglotAccess.ALL) maybeOutputStream.foreach(os => ctxBuilder.out(os)) diff --git a/python-client/src/test/scala/raw/client/python/TestPythonCompilerService.scala b/python-client/src/test/scala/raw/client/python/TestPythonCompilerService.scala index f38f8ecc0..95e6a3a93 100644 --- a/python-client/src/test/scala/raw/client/python/TestPythonCompilerService.scala +++ b/python-client/src/test/scala/raw/client/python/TestPythonCompilerService.scala @@ -13,7 +13,7 @@ package raw.client.python import raw.client.api.{CompilerService, ExecutionSuccess, ProgramEnvironment, RawInt} -import raw.utils.{InteractiveUser, RawTestSuite, SettingsTestContext, TrainingWheelsContext, Uid} +import raw.utils.{RawTestSuite, SettingsTestContext, TrainingWheelsContext, RawUid} import java.io.ByteArrayOutputStream @@ -21,7 +21,7 @@ class TestPythonCompilerService extends RawTestSuite with SettingsTestContext wi var compilerService: CompilerService = _ - val user = InteractiveUser(Uid("uid"), "name", "email", Seq.empty) + val user = RawUid("uid") override def beforeAll(): Unit = { super.beforeAll() @@ -38,21 +38,48 @@ class TestPythonCompilerService extends RawTestSuite with SettingsTestContext wi } test("basic execute test") { _ => - val environment = ProgramEnvironment(user, None, Set.empty, Map("output-format" -> "json")) + val environment = ProgramEnvironment( + user, + None, + Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map("output-format" -> "json") + ) val baos = new ByteArrayOutputStream() assert(compilerService.execute("1+1", environment, None, baos) == ExecutionSuccess(true)) assert(baos.toString() == "2") } test("basic execute test w/ decl") { _ => - val environment = ProgramEnvironment(user, None, Set.empty, Map("output-format" -> "json")) + val environment = ProgramEnvironment( + user, + None, + Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map("output-format" -> "json") + ) val baos = new ByteArrayOutputStream() assert(compilerService.execute("def f(): return 1+1", environment, Some("f"), baos) == ExecutionSuccess(true)) assert(baos.toString() == "2") } test("basic execute test w/ decl and arguments") { _ => - val environment = ProgramEnvironment(user, Some(Array("v" -> RawInt(2))), Set.empty, Map("output-format" -> "json")) + val environment = ProgramEnvironment( + user, + Some(Array("v" -> RawInt(2))), + Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map("output-format" -> "json") + ) val baos = new ByteArrayOutputStream() assert(compilerService.execute("def f(v): return v*2", environment, Some("f"), baos) == ExecutionSuccess(true)) assert(baos.toString() == "4") diff --git a/snapi-client/src/main/scala/raw/client/rql2/api/Rql2CompilerService.scala b/snapi-client/src/main/scala/raw/client/rql2/api/Rql2CompilerService.scala index 05ed567b6..a680e6cac 100644 --- a/snapi-client/src/main/scala/raw/client/rql2/api/Rql2CompilerService.scala +++ b/snapi-client/src/main/scala/raw/client/rql2/api/Rql2CompilerService.scala @@ -15,13 +15,13 @@ package raw.client.rql2.api import raw.client.api.{CompilerService, Message, ProgramEnvironment} import raw.compiler.base.source.{BaseNode, Type} import raw.compiler.common.source.SourceProgram -import raw.utils.AuthenticatedUser +import raw.utils.RawUid trait Rql2CompilerService extends CompilerService { - def prettyPrint(node: BaseNode, user: AuthenticatedUser): String + def prettyPrint(node: BaseNode, user: RawUid): String - def parseType(tipe: String, user: AuthenticatedUser, internal: Boolean = false): ParseTypeResponse + def parseType(tipe: String, user: RawUid, internal: Boolean = false): ParseTypeResponse def parse(source: String, environment: ProgramEnvironment): ParseResponse diff --git a/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala b/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala index bd14181cf..8ac97bb3a 100644 --- a/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala +++ b/snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2TruffleCompilerService.scala @@ -29,10 +29,8 @@ import raw.compiler.rql2.builtin.{BinaryPackage, CsvPackage, JsonPackage, String import raw.compiler.rql2.errors._ import raw.compiler.rql2.lsp.CompilerLspService import raw.compiler.rql2.source._ -import raw.creds.api.CredentialsServiceProvider import raw.inferrer.api.InferrerServiceProvider -import raw.sources.api.SourceContext -import raw.utils.{AuthenticatedUser, RawSettings, RawUtils} +import raw.utils.{RawSettings, RawUid, RawUtils} import java.io.{IOException, OutputStream} import scala.collection.mutable @@ -72,39 +70,34 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p override def language: Set[String] = Rql2TruffleCompilerService.LANGUAGE - private val credentials = CredentialsServiceProvider() - // Map of users to compiler context. - private val compilerContextCaches = new mutable.HashMap[AuthenticatedUser, CompilerContext] + private val compilerContextCaches = new mutable.HashMap[RawUid, CompilerContext] private val compilerContextCachesLock = new Object - private def getCompilerContext(user: AuthenticatedUser): CompilerContext = { + private def getCompilerContext(user: RawUid): CompilerContext = { compilerContextCachesLock.synchronized { compilerContextCaches.getOrElseUpdate(user, createCompilerContext(user, "rql2-truffle")) } } - private def createCompilerContext(user: AuthenticatedUser, language: String): CompilerContext = { - // Initialize source context - implicit val sourceContext = new SourceContext(user, credentials, settings) - + private def createCompilerContext(user: RawUid, language: String): CompilerContext = { // Initialize inferrer val inferrer = InferrerServiceProvider() // Initialize compiler context - new CompilerContext(language, user, inferrer, sourceContext) + new CompilerContext(language, user, inferrer) } - private def getProgramContext(user: AuthenticatedUser, environment: ProgramEnvironment): ProgramContext = { + private def getProgramContext(user: RawUid, environment: ProgramEnvironment): ProgramContext = { val compilerContext = getCompilerContext(user) new Rql2ProgramContext(environment, compilerContext) } - override def prettyPrint(node: BaseNode, user: AuthenticatedUser): String = { + override def prettyPrint(node: BaseNode, user: RawUid): String = { SourcePrettyPrinter.format(node) } - override def parseType(tipe: String, user: AuthenticatedUser, internal: Boolean = false): ParseTypeResponse = { + override def parseType(tipe: String, user: RawUid, internal: Boolean = false): ParseTypeResponse = { val positions = new Positions() val parser = new Antlr4SyntaxAnalyzer(positions, !internal) parser.parseType(tipe) match { @@ -114,7 +107,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p } override def parse(source: String, environment: ProgramEnvironment): ParseResponse = { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { val positions = new Positions() val parser = new Antlr4SyntaxAnalyzer(positions, true) @@ -136,7 +129,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { val tree = new TreeWithPositions(source, ensureTree = false, frontend = true)(programContext) if (tree.valid) { @@ -158,7 +151,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { val tree = new TreeWithPositions(source, ensureTree = false, frontend = true)(programContext) if (tree.valid) { @@ -218,7 +211,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p // its type is found in the polyglot bindings as '@type:' val funType = { val rawType = ctx.getPolyglotBindings.getMember("@type:" + decl).asString() - val ParseTypeSuccess(tipe: FunType) = parseType(rawType, environment.user, internal = true) + val ParseTypeSuccess(tipe: FunType) = parseType(rawType, environment.uid, internal = true) tipe } // Prior to .execute, some checks on parameters since we may have @@ -263,7 +256,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p val result = ctx.eval(truffleSource) // the value type is found in polyglot bindings after calling eval(). val rawType = ctx.getPolyglotBindings.getMember("@type").asString() - val ParseTypeSuccess(tipe) = parseType(rawType, environment.user, internal = true) + val ParseTypeSuccess(tipe) = parseType(rawType, environment.uid, internal = true) (result, tipe) } @@ -274,7 +267,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p if (!CsvPackage.outputWriteSupport(tipe)) { return ExecutionRuntimeFailure("unsupported type") } - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) val windowsLineEnding = environment.options.get("windows-line-ending") match { case Some("true") => true case _ => programContext.settings.config.getBoolean("raw.compiler.windows-line-ending") @@ -346,7 +339,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p if (ex.isInternalError) { // An internal error. It means a regular Exception thrown from the language (e.g. a Java Exception, // or a RawTruffleInternalErrorException, which isn't an AbstractTruffleException) - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) throw new CompilerServiceException(ex, programContext.dumpDebugInfo) } else { val err = ex.getGuestObject @@ -391,7 +384,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p maybeIndent: Option[Int], maybeWidth: Option[Int] ): FormatCodeResponse = { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { val pretty = new SourceCommentsPrettyPrinter(maybeIndent, maybeWidth) pretty.prettyCode(source) match { @@ -411,7 +404,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree(source, lspService => lspService.dotAutoComplete(source, environment, position))( programContext @@ -435,7 +428,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree(source, lspService => lspService.wordAutoComplete(source, environment, prefix, position))( programContext @@ -460,7 +453,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree(source, lspService => lspService.hover(source, environment, position))(programContext) match { case Right(value) => value @@ -477,7 +470,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree(source, lspService => lspService.rename(source, environment, position))(programContext) match { case Right(value) => value @@ -498,7 +491,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree(source, lspService => lspService.definition(source, environment, position))( programContext @@ -517,7 +510,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) try { withLspTree( source, @@ -537,7 +530,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p withTruffleContext( environment, _ => { - val programContext = getProgramContext(environment.user, environment) + val programContext = getProgramContext(environment.uid, environment) // Will analyze the code and return only unknown declarations errors. val positions = new Positions() val parser = new Antlr4SyntaxAnalyzer(positions, true) @@ -612,7 +605,6 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p override def doStop(): Unit = { compilerContextCaches.values.foreach(compilerContext => compilerContext.inferrer.stop()) - credentials.stop() if (initedEngine) { CompilerService.releaseEngine } @@ -651,9 +643,7 @@ class Rql2TruffleCompilerService(engineDefinition: (Engine, Boolean))(implicit p val ctxBuilder = Context .newBuilder("rql") .engine(engine) - .environment("RAW_USER", environment.user.uid.toString) - .environment("RAW_TRACE_ID", environment.user.uid.toString) - .environment("RAW_SCOPES", environment.scopes.mkString(",")) + .environment("RAW_PROGRAM_ENVIRONMENT", ProgramEnvironment.serializeToString(environment)) .allowExperimentalOptions(true) .allowPolyglotAccess(PolyglotAccess.ALL) environment.options.get("staged-compiler").foreach { stagedCompiler => diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/Rql2CompilerTestContext.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/Rql2CompilerTestContext.scala index 423df52e2..15c33dfb6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/Rql2CompilerTestContext.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/Rql2CompilerTestContext.scala @@ -29,6 +29,90 @@ import java.nio.file.{Files, Path, StandardOpenOption} import scala.collection.mutable import scala.io.Source +object TestCredentials { + + private def getEnv(name: String): String = sys.env.getOrElse(name, "credential_not_defined_missing_env_file") + + ///////////////////////////////////////////////////////////////////////////// + // HTTP Headers + ///////////////////////////////////////////////////////////////////////////// + + val dropboxLongLivedAccessToken = getEnv("RAW_DROPBOX_TEST_LONG_LIVED_ACCESS_TOKEN") + // The client ID to use for Dropbox API calls, once the access token is obtained. + val dropboxClientId = getEnv("RAW_DROPBOX_TEST_CLIENT_ID") + + ///////////////////////////////////////////////////////////////////////////// + // S3 Credentials + ///////////////////////////////////////////////////////////////////////////// + + val accessKeyId = getEnv("RAW_AWS_ACCESS_KEY_ID") + val secretKeyId = getEnv("RAW_AWS_SECRET_ACCESS_KEY") + + // Bucket with public access + val UnitTestPublicBucket = "rawlabs-public-test-data" + val UnitTestPublicBucketCred = S3Credential(None, None, Some("eu-west-1")) + + // IAM user 'unit-test-private-bucket', which only has permissions only to access bucket 'rawlabs-private-test-data' + val UnitTestPrivateBucket = "rawlabs-private-test-data" + val UnitTestPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1")) + + val UnitTestPrivateBucket2 = "rawlabs-unit-tests" + val UnitTestPrivateBucket2Cred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1")) + + val UnitTestEmptyBucketPrivateBucket = "rawlabs-unit-test-empty-bucket" + val UnitTestEmptyBucketPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1")) + + val UnitTestListRootPrivateBucket = "rawlabs-unit-test-list-root" + val UnitTestListRootPrivateBucketCred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("eu-west-1")) + + val unitTestPrivateBucketUsEast1 = "rawlabs-unit-tests-us-east-1" + val unitTestPrivateBucketUsEast1Cred = S3Credential(Some(accessKeyId), Some(secretKeyId), Some("us-east-1")) + + /////////////////////////////////////////////////////////////////////////// + // Jdbc Credentials + /////////////////////////////////////////////////////////////////////////// + + val mysqlTestHost = getEnv("RAW_MYSQL_TEST_HOST") + val mysqlTestDB = getEnv("RAW_MYSQL_TEST_DB") + val mysqlTestUser = getEnv("RAW_MYSQL_TEST_USER") + val mysqlTestPassword = getEnv("RAW_MYSQL_TEST_PASSWORD") + val mysqlCreds = MySqlJdbcLocation(mysqlTestHost, 3306, mysqlTestDB, mysqlTestUser, mysqlTestPassword) + val pgsqlTestHost = getEnv("RAW_PGSQL_TEST_HOST") + val pgsqlTestDB = getEnv("RAW_PGSQL_TEST_DB") + val pgsqlTestUser = getEnv("RAW_PGSQL_TEST_USER") + val pgsqlTestPassword = getEnv("RAW_PGSQL_TEST_PASSWORD") + val pgsqlCreds = PostgresJdbcLocation(pgsqlTestHost, 5432, pgsqlTestDB, pgsqlTestUser, pgsqlTestPassword) + val oracleTestHost = getEnv("RAW_ORACLE_TEST_HOST") + val oracleTestDB = getEnv("RAW_ORACLE_TEST_DB") + val oracleTestUser = getEnv("RAW_ORACLE_TEST_USER") + val oracleTestPassword = getEnv("RAW_ORACLE_TEST_PASSWORD") + val oracleCreds = OracleJdbcLocation(oracleTestHost, 1521, oracleTestDB, oracleTestUser, oracleTestPassword) + val sqlServerTestHost = getEnv("RAW_SQLSERVER_TEST_HOST") + val sqlserverTestDB = getEnv("RAW_SQLSERVER_TEST_DB") + val sqlServerTestUser = getEnv("RAW_SQLSERVER_TEST_USER") + val sqlServerTestPassword = getEnv("RAW_SQLSERVER_TEST_PASSWORD") + val sqlServerCreds = SqlServerJdbcLocation( + sqlServerTestHost, + 1433, + sqlserverTestDB, + sqlServerTestUser, + sqlServerTestPassword + ) + val snowflakeTestHost = getEnv("RAW_SNOWFLAKE_TEST_HOST") + val snowflakeTestDB = getEnv("RAW_SNOWFLAKE_TEST_DB") + val snowflakeTestUser = getEnv("RAW_SNOWFLAKE_TEST_USER") + val snowflakeTestPassword = getEnv("RAW_SNOWFLAKE_TEST_PASSWORD") + val snowflakeCreds = SnowflakeJdbcLocation( + snowflakeTestDB, + snowflakeTestUser, + snowflakeTestPassword, + snowflakeTestHost, + Map("timezone" -> "UTC") + ) + val badMysqlCreds = MySqlJdbcLocation("does-not-exist.raw-labs.com", 3306, "rdbmstest", "t0or", "$up3r$3cr3tValu3") + +} + trait Rql2CompilerTestContext extends RawTestSuite with Matchers @@ -40,7 +124,17 @@ trait Rql2CompilerTestContext // Simple inferrer with LocalInferrerTestContext { - def authorizedUser: InteractiveUser = InteractiveUser(Uid("janeUid"), "Jane Smith", "jane@example.com") + private val secrets = new mutable.HashMap[String, String]() + + private val s3Credentials = new mutable.HashMap[String, S3Credential]() + + private val rdbmsServers = new mutable.HashMap[String, JdbcLocation]() + + private val credHttpHeaders = new mutable.HashMap[String, Map[String, String]]() + + protected val programOptions = new mutable.HashMap[String, String]() + + def authorizedUser: RawUid = RawUid("janeUid") def runnerScopes: Set[String] = Set.empty @@ -48,7 +142,21 @@ trait Rql2CompilerTestContext def maybeTraceId: Option[String] = None - protected val programOptions = new mutable.HashMap[String, String]() + def secret(name: String, value: String): Unit = { + secrets.put(name, value) + } + + def s3Bucket(name: String, bucket: S3Credential): Unit = { + s3Credentials.put(name, bucket) + } + + def rdbms(name: String, db: JdbcLocation): Unit = { + rdbmsServers.put(name, db) + } + + def httpHeaders(name: String, headers: Map[String, String]): Unit = { + credHttpHeaders.put(name, headers) + } def option(key: String, value: String): Unit = { programOptions.put(key, value) @@ -468,13 +576,21 @@ trait Rql2CompilerTestContext maybeArguments: Option[Array[(String, RawValue)]] = None, scopes: Set[String] = Set.empty, options: Map[String, String] = Map.empty - ): ProgramEnvironment = ProgramEnvironment( - authorizedUser, - maybeArguments, - this.runnerScopes ++ scopes, - this.options ++ options ++ programOptions, - maybeTraceId - ) + ): ProgramEnvironment = { + val user = authorizedUser + ProgramEnvironment( + user, + maybeArguments, + this.runnerScopes ++ scopes, + secrets.toMap, + rdbmsServers.toMap, + credHttpHeaders.toMap, + s3Credentials.toMap, + this.options ++ options ++ programOptions, + None, // jdbcUrl + maybeTraceId + ) + } def parseQuery(code: String): BaseProgram = tryToParse(code) match { case Right(p) => p diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/BenchmarkTests.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/BenchmarkTests.scala index a4cf61189..ec9366a6b 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/BenchmarkTests.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/BenchmarkTests.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.benchmark -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BenchmarkTests extends Rql2CompilerTestContext { +@TruffleTests class BenchmarkTests extends Rql2TruffleCompilerTestContext { property("raw.training-wheels", "false") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/StressTests.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/StressTests.scala index cfd25ebe7..96e5dadd6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/StressTests.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/benchmark/StressTests.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.benchmark -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait StressTests extends Rql2CompilerTestContext { +@TruffleTests class StressTests extends Rql2TruffleCompilerTestContext { val shouldBeExecuted = false diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BinaryPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BinaryPackageTest.scala index a5d5a86a8..55188cf8c 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BinaryPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BinaryPackageTest.scala @@ -13,12 +13,13 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path import java.util.Base64 -trait BinaryPackageTest extends Rql2CompilerTestContext { +@TruffleTests class BinaryPackageTest extends Rql2TruffleCompilerTestContext { // FIXME (msb): This should use cast to support string to binary and do .getBytes("utf-8") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BytePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BytePackageTest.scala index 5275b2d86..293dcb27a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BytePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/BytePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BytePackageTest extends Rql2CompilerTestContext { +@TruffleTests class BytePackageTest extends Rql2TruffleCompilerTestContext { test(""" Byte.From(1)""")(it => it should evaluateTo("1b")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/CsvPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/CsvPackageTest.scala index dd1d51923..1b66efbde 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/CsvPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/CsvPackageTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait CsvPackageTest extends Rql2CompilerTestContext { +@TruffleTests class CsvPackageTest extends Rql2TruffleCompilerTestContext { val ttt = "\"\"\"" @@ -58,13 +59,13 @@ trait CsvPackageTest extends Rql2CompilerTestContext { test(snapi"""Csv.Read("$badData", type collection(record(a: int, b: int, c: int, d: double)), delimiter="|")""")(it => if (isTruffle) { it should runErrorAs( - snapi"failed to read CSV (line 2 column 9) (url: $badData): not enough columns found" + snapi"failed to read CSV (line 2 column 9) (location: $badData): not enough columns found" ) } else { // scala engine: columns are given as "CSV column index" instead of "character index" // + duplicates the location it should runErrorAs( - snapi"""failed to read CSV (line 2 column 4) (url: $badData): failed to parse CSV (line 2, col 4), not enough columns found""" + snapi"""failed to read CSV (line 2 column 4) (location: $badData): failed to parse CSV (line 2, col 4), not enough columns found""" ) } ) @@ -72,13 +73,13 @@ trait CsvPackageTest extends Rql2CompilerTestContext { test(snapi"""Csv.Read("$badData", type collection(record(a: int, b: int, c: int, d: double)), delimiter=";")""")(it => if (isTruffle) { it should runErrorAs( - snapi"failed to read CSV (line 1 column 14) (url: $badData): not enough columns found" + snapi"failed to read CSV (line 1 column 14) (location: $badData): not enough columns found" ) } else { // scala engine: columns are given as "CSV column index" instead of "character index" // + duplicates the location it should runErrorAs( - snapi"""failed to read CSV (line 1 column 2) (url: $badData): failed to parse CSV (line 1, col 2), not enough columns found""" + snapi"""failed to read CSV (line 1 column 2) (location: $badData): failed to parse CSV (line 1, col 2), not enough columns found""" ) } ) @@ -240,9 +241,9 @@ trait CsvPackageTest extends Rql2CompilerTestContext { if (isTruffle) { it should evaluateTo( snapi"""[{ - | a : Error.Build("failed to parse CSV (url: $csvWithNulls: line 1002, col 1), cannot parse 'NA' as an int"), + | a : Error.Build("failed to parse CSV (location: $csvWithNulls: line 1002, col 1), cannot parse 'NA' as an int"), | b : "NA", // it's not parsed as a null in that test since we didn't pass it in the nulls list - | c : Error.Build("failed to parse CSV (url: $csvWithNulls: line 1002, col 7), cannot parse 'NA' as a double") + | c : Error.Build("failed to parse CSV (location: $csvWithNulls: line 1002, col 7), cannot parse 'NA' as a double") |}] """.stripMargin ) } else { @@ -272,9 +273,9 @@ trait CsvPackageTest extends Rql2CompilerTestContext { if (isTruffle) { it should evaluateTo( snapi"""[{ - | a : Error.Build("failed to parse CSV (url: $csvWithTypeChange: line 1002, col 1), cannot parse 'hello' as an int"), + | a : Error.Build("failed to parse CSV (location: $csvWithTypeChange: line 1002, col 1), cannot parse 'hello' as an int"), | b : null, - | c : Error.Build("failed to parse CSV (url: $csvWithTypeChange: line 1002, col 8), cannot parse 'world' as a double") + | c : Error.Build("failed to parse CSV (location: $csvWithTypeChange: line 1002, col 8), cannot parse 'world' as a double") |}] """.stripMargin ) } else { @@ -295,11 +296,11 @@ trait CsvPackageTest extends Rql2CompilerTestContext { // ideally it would be line 11, column 30 (end of line 11) if (isTruffle) { it should runErrorAs( - snapi"failed to read CSV (line 11 column 173) (url: $junkAfter10Items): not enough columns found" + snapi"failed to read CSV (line 11 column 173) (location: $junkAfter10Items): not enough columns found" ) } else { it should runErrorAs( - snapi"failed to read CSV (line 10 column 2) (url: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" + snapi"failed to read CSV (line 10 column 2) (location: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" ) } } @@ -326,11 +327,11 @@ trait CsvPackageTest extends Rql2CompilerTestContext { // ideally it would be line 11, column 30 (end of line 11) if (isTruffle) { it should runErrorAs( - snapi"failed to read CSV (line 11 column 173) (url: $junkAfter10Items): not enough columns found" + snapi"failed to read CSV (line 11 column 173) (location: $junkAfter10Items): not enough columns found" ) } else { it should runErrorAs( - snapi"failed to read CSV (line 10 column 2) (url: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" + snapi"failed to read CSV (line 10 column 2) (location: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" ) } } @@ -341,11 +342,11 @@ trait CsvPackageTest extends Rql2CompilerTestContext { // ideally it would be line 11, column 30 (end of line 11) if (isTruffle) { it should runErrorAs( - snapi"failed to read CSV (line 11 column 173) (url: $junkAfter10Items): not enough columns found" + snapi"failed to read CSV (line 11 column 173) (location: $junkAfter10Items): not enough columns found" ) } else { it should runErrorAs( - snapi"failed to read CSV (line 10 column 2) (url: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" + snapi"failed to read CSV (line 10 column 2) (location: $junkAfter10Items): failed to parse CSV (line 10, col 2), not enough columns found" ) } } @@ -447,17 +448,17 @@ trait CsvPackageTest extends Rql2CompilerTestContext { |), delimiter = ";", skip = 0)""".stripMargin)(it => if (isTruffle) { it should evaluateTo(snapi"""[ - |{byteCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 1), cannot parse 'byteCol' as a byte"), - | shortCol:Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 9), cannot parse 'shortCol' as a short"), - | intCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 18), cannot parse 'intCol' as an int"), - | longCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 25), cannot parse 'longCol' as a long"), - | floatCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 33), cannot parse 'floatCol' as a float"), - | doubleCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 42), cannot parse 'doubleCol' as a double"), - | decimalCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 52), cannot parse 'decimalCol' as a decimal"), - | boolCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 63), cannot parse 'boolCol' as a bool"), - | dateCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 71), string 'dateCol' does not match date template 'yyyy-M-d'"), - | timeCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 79), string 'timeCol' does not match time template 'HH:mm[:ss[.SSS]]'"), - | timestampCol: Error.Build("failed to parse CSV (url: $csvWithAllTypes: line 1, col 87), string 'timestampCol' does not match timestamp template 'HH:mm[:ss[.SSS]]'")}, + |{byteCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 1), cannot parse 'byteCol' as a byte"), + | shortCol:Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 9), cannot parse 'shortCol' as a short"), + | intCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 18), cannot parse 'intCol' as an int"), + | longCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 25), cannot parse 'longCol' as a long"), + | floatCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 33), cannot parse 'floatCol' as a float"), + | doubleCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 42), cannot parse 'doubleCol' as a double"), + | decimalCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 52), cannot parse 'decimalCol' as a decimal"), + | boolCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 63), cannot parse 'boolCol' as a bool"), + | dateCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 71), string 'dateCol' does not match date template 'yyyy-M-d'"), + | timeCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 79), string 'timeCol' does not match time template 'HH:mm[:ss[.SSS]]'"), + | timestampCol: Error.Build("failed to parse CSV (location: $csvWithAllTypes: line 1, col 87), string 'timestampCol' does not match timestamp template 'HH:mm[:ss[.SSS]]'")}, |{byteCol: Byte.From(1), shortCol:Short.From(10), intCol: Int.From(100), longCol: Long.From(1000), | floatCol: Float.From(3.14), doubleCol: Double.From(6.28), decimalCol: Decimal.From("9.42"), boolCol: true, | dateCol: Date.Parse("12/25/2023", "M/d/yyyy"), timeCol: Time.Parse("01:02:03", "H:m:s"), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DatePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DatePackageTest.scala index 942049d2d..aa17d0b37 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DatePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DatePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait DatePackageTest extends Rql2CompilerTestContext { +@TruffleTests class DatePackageTest extends Rql2TruffleCompilerTestContext { test("Date.Build(2022, 1, 15)") { it => it should typeAs("date") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DecimalPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DecimalPackageTest.scala index 337ab7504..e3e302b98 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DecimalPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DecimalPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait DecimalPackageTest extends Rql2CompilerTestContext { +@TruffleTests class DecimalPackageTest extends Rql2TruffleCompilerTestContext { test("""Decimal.Round(Decimal.From("1.423"), 2)""") { it => it should evaluateTo("""1.42q""") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DoublePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DoublePackageTest.scala index 5cef07731..132cabd6a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DoublePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/DoublePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait DoublePackageTest extends Rql2CompilerTestContext { +@TruffleTests class DoublePackageTest extends Rql2TruffleCompilerTestContext { test(""" Double.From(1)""")(it => it should evaluateTo("1.0")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/EnvironmentPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/EnvironmentPackageTest.scala index 33c252a6b..e80068d19 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/EnvironmentPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/EnvironmentPackageTest.scala @@ -13,13 +13,14 @@ package raw.compiler.rql2.tests.builtin import com.google.common.collect.HashMultiset -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import scala.collection.JavaConverters._ -trait EnvironmentPackageTest extends Rql2CompilerTestContext { +@TruffleTests class EnvironmentPackageTest extends Rql2TruffleCompilerTestContext { - test("""Environment.Secret("my-typo")""")(it => it should runErrorAs("could not find secret my-typo")) + test("""Environment.Secret("my-typo")""")(it => it should runErrorAs("unknown secret: my-typo")) test("""Environment.Scopes()""")(it => it should evaluateTo("""[]""")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ErrorPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ErrorPackageTest.scala index f28950b53..cb203f585 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ErrorPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ErrorPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ErrorPackageTest extends Rql2CompilerTestContext { +@TruffleTests class ErrorPackageTest extends Rql2TruffleCompilerTestContext { test(""" Error.Build("an error")""")(it => it should runErrorAs("an error")) test(""" Error.Build("an error") == Error.Build("an error")""")(it => it should runErrorAs("an error")) test(""" Error.Build("an error") == Error.Build("another error")""")(it => it should runErrorAs("an error")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FloatPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FloatPackageTest.scala index fc2b54273..e3b5da877 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FloatPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FloatPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait FloatPackageTest extends Rql2CompilerTestContext { +@TruffleTests class FloatPackageTest extends Rql2TruffleCompilerTestContext { test(""" Float.From(1)""")(it => it should evaluateTo("1.0f")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FunctionPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FunctionPackageTest.scala index b0404de7d..54ed43612 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FunctionPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/FunctionPackageTest.scala @@ -13,10 +13,10 @@ package raw.compiler.rql2.tests.builtin import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.sources.bytestream.api.HttpLocationsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait FunctionPackageTest extends Rql2CompilerTestContext with HttpLocationsTestContext { +@TruffleTests class FunctionPackageTest extends Rql2TruffleCompilerTestContext { test("""Function.InvokeAfter(() -> 1 +1, 10)""".stripMargin)(_ should evaluateTo("2")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/HttpPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/HttpPackageTest.scala index ddd3a1108..3991049ac 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/HttpPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/HttpPackageTest.scala @@ -11,16 +11,16 @@ */ package raw.compiler.rql2.tests.builtin + import com.sun.net.httpserver.{BasicAuthenticator, HttpExchange, HttpServer} import com.typesafe.scalalogging.StrictLogging -import org.scalatest.BeforeAndAfterAll -import raw.creds.dropbox.DropboxTestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.net.InetSocketAddress import scala.collection.JavaConverters._ -trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with BeforeAndAfterAll { +@TruffleTests class HttpPackageTest extends Rql2TruffleCompilerTestContext { val expectedUser = "user" val expectedPassword = "passwd" @@ -136,10 +136,7 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Get("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "GET" - | ) + | Http.Get("http://localhost:$testPort/hello") |)""".stripMargin) it should evaluateTo(""" "Hello World" """) } @@ -149,10 +146,7 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Post("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "POST" - | ) + | Http.Post("http://localhost:$testPort/hello") |)""".stripMargin) it should evaluateTo(""" "Hello World" """) } @@ -162,9 +156,8 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Put("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "PUT" + | Http.Put( + | "http://localhost:$testPort/hello" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -175,9 +168,8 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Delete("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "DELETE" + | Http.Delete( + | "http://localhost:$testPort/hello" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -189,9 +181,8 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Head("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "HEAD" + | Http.Head( + | "http://localhost:$testPort/hello" | ) |)""".stripMargin) } @@ -201,9 +192,8 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Options("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "OPTIONS" + | Http.Options( + | "http://localhost:$testPort/hello" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -214,9 +204,8 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | Http.Patch("http://localhost:$testPort/hello") |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/hello", - | http_method = "PATCH" + | Http.Patch( + | "http://localhost:$testPort/hello" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -232,12 +221,11 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | ) |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( + | Http.Post( | "http://localhost:$testPort/return-body", - | http_method = "Post", - | http_user_name = "$expectedUser", - | http_password = "$expectedPassword", - | http_body_string = "Hello World" + | username = "$expectedUser", + | password = "$expectedPassword", + | bodyString = "Hello World" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -253,12 +241,11 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with | ) |)""".stripMargin) { it => it should evaluateTo(s"""String.Read( - | Location.Build( + | Http.Post( | "http://localhost:$testPort/return-body", - | http_method = "Post", - | http_user_name = "$expectedUser", - | http_password = "$expectedPassword", - | http_body_string = "Hello World" + | username = "$expectedUser", + | password = "$expectedPassword", + | bodyString = "Hello World" | ) |)""".stripMargin) it should evaluateTo(""" "Hello World" """) @@ -300,7 +287,7 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with |""".stripMargin)(it => it should evaluateTo(s""" |List.Build( | Record.Build(url="http://localhost:$wrongPort/hello", - | r=Error.Build("http error: host not found for http://localhost:$wrongPort/hello")), + | r=Error.Build("host not found for http://localhost:$wrongPort/hello")), | Record.Build(url="http://localhost:$testPort/hello", r=200), | Record.Build(url="http://localhost:$testPort/darkness", r=404) |)""".stripMargin)) @@ -340,31 +327,16 @@ trait HttpPackageTest extends Rql2CompilerTestContext with DropboxTestCreds with |)""".stripMargin)(it => it should evaluateTo(s""" ${triple}Header1:value1 |Header2:value2$triple """.stripMargin)) - test(s"""String.Read( - | Location.Build( - | "http://localhost:$testPort/return-headers", - | http_headers = [ - | {"Header1", "value1"}, - | {"Header2", "value2"} - | ] - | ) - |)""".stripMargin)(it => - it should evaluateTo( - s""" ${triple}Header1:value1 - |Header2:value2$triple """.stripMargin - ) - ) - // Restricted headers are properly leading to an error (RD-6871) test(s"""String.Read( - | Location.Build( + | Http.Get( | "http://localhost:$testPort/return-headers", - | http_headers = [ + | headers = [ | {"Host", "value1"}, | {"Header2", "value2"} | ] | ) - |)""".stripMargin)(it => it should runErrorAs("http error: restricted header name: \"Host\"")) + |)""".stripMargin)(it => it should runErrorAs("restricted header name: \"Host\"")) test(s"""let | query = $triple SELECT DISTINCT (?country as ?wikidata_country) ?countryLabel ?code diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntPackageTest.scala index 0abf153e0..90547421d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait IntPackageTest extends Rql2CompilerTestContext { +@TruffleTests class IntPackageTest extends Rql2TruffleCompilerTestContext { test(""" Int.From(1)""")(it => it should evaluateTo("1")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntervalPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntervalPackageTest.scala index 746a30605..56d16a474 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntervalPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/IntervalPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait IntervalPackageTest extends Rql2CompilerTestContext { +@TruffleTests class IntervalPackageTest extends Rql2TruffleCompilerTestContext { test("Interval.Build(months=12) == Interval.Build(years=1)")(_ should evaluateTo("true")) test("Interval.Build(hours=24) == Interval.Build(days=1)")(_ should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/JsonPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/JsonPackageTest.scala index 08e195842..c4767d2bb 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/JsonPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/JsonPackageTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait JsonPackageTest extends Rql2CompilerTestContext { +@TruffleTests class JsonPackageTest extends Rql2TruffleCompilerTestContext { private val junkAfter10Items = tempFile("""[ | {"a": 1, "b": "#1", "c": 1.1}, @@ -169,7 +170,7 @@ trait JsonPackageTest extends Rql2CompilerTestContext { )(it => it should typeErrorAs("unsupported type")) test( - """Json.Print(Location.Build("http://something"))""" + """Json.Print(Http.Get("http://something"))""" )(it => it should typeErrorAs("unsupported type")) test(snapi"""Json.InferAndRead("$data")""".stripMargin)(it => it should run) @@ -422,11 +423,11 @@ trait JsonPackageTest extends Rql2CompilerTestContext { test(snapi"""Json.Read("$junkAfter10Items", type collection(record(a: int, b: string, c: double)))""") { it => if (isTruffle) { it should runErrorAs( - snapi"failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35))" + snapi"failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35))" ) } else { it should runErrorAs( - snapi"""failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries + snapi"""failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries | at [Source: (InputStreamReader); line: 11, column: 37]""".stripMargin ) } @@ -453,11 +454,11 @@ trait JsonPackageTest extends Rql2CompilerTestContext { ) { it => if (isTruffle) { it should runErrorAs( - snapi"failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35))" + snapi"failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35))" ) } else { it should runErrorAs( - snapi"""failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries + snapi"""failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries | at [Source: (InputStreamReader); line: 11, column: 37]""".stripMargin ) } @@ -468,11 +469,11 @@ trait JsonPackageTest extends Rql2CompilerTestContext { ) { it => if (isTruffle) { it should runErrorAs( - snapi"failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35))" + snapi"failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35))" ) } else { it should runErrorAs( - snapi"""failed to read JSON (line 11 column 37) (url: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries + snapi"""failed to read JSON (line 11 column 37) (location: $junkAfter10Items): Unexpected character ('#' (code 35)): was expecting comma to separate Array entries | at [Source: (InputStreamReader); line: 11, column: 37]""".stripMargin ) } @@ -614,7 +615,7 @@ trait JsonPackageTest extends Rql2CompilerTestContext { snapi"""Json.Read("$xmlFile", type collection(record(name: string, birthYear: int)))""" ) { it => it should runErrorAs( - s"failed to read JSON (line 1 column 2) (url: file:${xmlFile.toAbsolutePath}): Unexpected character ('<' (code 60)): expected a valid value (JSON String, Number (or 'NaN'/'INF'/'+INF'), Array, Object or token 'null', 'true' or 'false')\n at [Source: (InputStreamReader); line: 1, column: 2]" + s"failed to read JSON (line 1 column 2) (location: file:${xmlFile.toAbsolutePath}): Unexpected character ('<' (code 60)): expected a valid value (JSON String, Number (or 'NaN'/'INF'/'+INF'), Array, Object or token 'null', 'true' or 'false')\n at [Source: (InputStreamReader); line: 1, column: 2]" ) } diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LocationPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LocationPackageTest.scala index c9fd33cc4..4a5c76fe3 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LocationPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LocationPackageTest.scala @@ -13,12 +13,13 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait LocationPackageTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class LocationPackageTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test(s""" |let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LongPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LongPackageTest.scala index 1dace17f7..03bff6726 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LongPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/LongPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LongPackageTest extends Rql2CompilerTestContext { +@TruffleTests class LongPackageTest extends Rql2TruffleCompilerTestContext { test(""" Long.From(1)""")(it => it should evaluateTo("1L")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/MathPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/MathPackageTest.scala index 36f5d18df..9b2fedeba 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/MathPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/MathPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait MathPackageTest extends Rql2CompilerTestContext { +@TruffleTests class MathPackageTest extends Rql2TruffleCompilerTestContext { // Nullable - Tryable tests test("""Math.Sin(if(true) then 3.13 else null)""")(_ should beCloseTo("0.011592393936158275")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullablePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullablePackageTest.scala index 78930ad1c..93452d690 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullablePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullablePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait NullablePackageTest extends Rql2CompilerTestContext { +@TruffleTests class NullablePackageTest extends Rql2TruffleCompilerTestContext { test("""Nullable.IsNull(null)""")(_ should evaluateTo("true")) test("""Nullable.IsNull(1)""")(_ should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullableTryablePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullableTryablePackageTest.scala index 20073725b..d7328f039 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullableTryablePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/NullableTryablePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait NullableTryablePackageTest extends Rql2CompilerTestContext { +@TruffleTests class NullableTryablePackageTest extends Rql2TruffleCompilerTestContext { test("""1 / null""")(_ should evaluateTo("null")) test(""" 1.0 / (if true then null else 2)""")(_ should evaluateTo("null")) test(""" 1.0 / (if false then null else 2)""")(_ should evaluateTo("0.5")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RecordPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RecordPackageTest.scala index 6222d0400..3ee63dfb0 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RecordPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RecordPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RecordPackageTest extends Rql2CompilerTestContext { +@TruffleTests class RecordPackageTest extends Rql2TruffleCompilerTestContext { test("""Record.Build(a = 1, b = "Hello")""".stripMargin)(_ should (typeAs("record(a: int, b: string)") and run)) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RegexPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RegexPackageTest.scala index 444528acf..11250b413 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RegexPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/RegexPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RegexPackageTest extends Rql2CompilerTestContext { +@TruffleTests class RegexPackageTest extends Rql2TruffleCompilerTestContext { test("""Regex.Replace("Heelloo John", "[aeiou]+", "_")""")(it => it should evaluateTo(""" "H_ll_ J_hn" """)) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/S3PackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/S3PackageTest.scala index cdae600d1..5881d24f2 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/S3PackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/S3PackageTest.scala @@ -12,10 +12,12 @@ package raw.compiler.rql2.tests.builtin -import raw.creds.s3.S3TestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait S3PackageTest extends Rql2CompilerTestContext with S3TestCreds { +@TruffleTests class S3PackageTest extends Rql2TruffleCompilerTestContext { + + import raw.compiler.rql2.tests.TestCredentials._ // reading a public s3 bucket without registering or passing credentials test(s"""let @@ -28,7 +30,7 @@ trait S3PackageTest extends Rql2CompilerTestContext with S3TestCreds { // reading a public s3 bucket without registering or passing credentials test(s"""let - | data = Csv.InferAndRead(S3.Build("s3://${UnitTestPublicBucket.name}/students.csv")) + | data = Csv.InferAndRead(S3.Build("s3://$UnitTestPublicBucket/students.csv")) |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ShortPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ShortPackageTest.scala index dc95b0758..1b456991a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ShortPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/ShortPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ShortPackageTest extends Rql2CompilerTestContext { +@TruffleTests class ShortPackageTest extends Rql2TruffleCompilerTestContext { test(""" Short.From(1)""")(it => it should evaluateTo("1s")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/StringPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/StringPackageTest.scala index a058bd40f..0121680bf 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/StringPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/StringPackageTest.scala @@ -14,11 +14,13 @@ package raw.compiler.rql2.tests.builtin import java.util.Base64 import raw.compiler.utils._ -import raw.compiler.rql2.tests.{FailAfterNServer, Rql2CompilerTestContext} +import raw.compiler.rql2.tests.FailAfterNServer +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait StringPackageTest extends Rql2CompilerTestContext with FailAfterNServer { +@TruffleTests class StringPackageTest extends Rql2TruffleCompilerTestContext with FailAfterNServer { // Each line has 11 bytes so it will fail at line 10 more or less. override def failServices: Seq[FailAfter] = Seq( @@ -248,15 +250,15 @@ trait StringPackageTest extends Rql2CompilerTestContext with FailAfterNServer { } test(s"""String.ReadLines("$testServerUrl/fail-after-10")""")( - _ should runErrorAs(s"failed to read lines (url: $testServerUrl/fail-after-10): closed") + _ should runErrorAs(s"failed to read lines (location: $testServerUrl/fail-after-10): closed") ) test(s"""Collection.Take(String.ReadLines("$testServerUrl/fail-after-10"), 11)""")( - _ should runErrorAs(s"failed to read lines (url: $testServerUrl/fail-after-10): closed") + _ should runErrorAs(s"failed to read lines (location: $testServerUrl/fail-after-10): closed") ) test(s"""Collection.Count(String.ReadLines("$testServerUrl/fail-after-10"))""".stripMargin)( - _ should runErrorAs(s"failed to read lines (url: $testServerUrl/fail-after-10): closed") + _ should runErrorAs(s"failed to read lines (location: $testServerUrl/fail-after-10): closed") ) test(s"""Try.IsError(Collection.Count(String.ReadLines("$testServerUrl/fail-after-10")) ) """) { diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/SuccessPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/SuccessPackageTest.scala index b974f781c..36052f3e1 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/SuccessPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/SuccessPackageTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.builtin import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait SuccessPackageTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class SuccessPackageTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("Success.Build(1)")(_ should evaluateTo("1")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimePackageTest.scala index 3b5fdedb1..d5ccd62fb 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimePackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait TimePackageTest extends Rql2CompilerTestContext { +@TruffleTests class TimePackageTest extends Rql2TruffleCompilerTestContext { test("Time.Build(9, 30)") { it => it should typeAs("time") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimestampPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimestampPackageTest.scala index 9ece622ee..97d25ebb7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimestampPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TimestampPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait TimestampPackageTest extends Rql2CompilerTestContext { +@TruffleTests class TimestampPackageTest extends Rql2TruffleCompilerTestContext { test("Timestamp.Build(2022, 1, 15, 9, 30)") { it => it should typeAs("timestamp") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TryPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TryPackageTest.scala index 027638cfa..c090ca2df 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TryPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TryPackageTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait TryPackageTest extends Rql2CompilerTestContext { +@TruffleTests class TryPackageTest extends Rql2TruffleCompilerTestContext { test("""Try.IsError("hi")""")(_ should evaluateTo("false")) test("""Try.IsError(1)""")(_ should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TypePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TypePackageTest.scala index 9ca79b867..2c782d0d7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TypePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/TypePackageTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait TypePackageTest extends Rql2CompilerTestContext { +@TruffleTests class TypePackageTest extends Rql2TruffleCompilerTestContext { ignore("""Types.Merge(int, float)""")(it => it should typeAs("float")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/XmlPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/XmlPackageTest.scala index 53502b548..c0f4e7281 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/XmlPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/XmlPackageTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait XmlPackageTest extends Rql2CompilerTestContext { +@TruffleTests class XmlPackageTest extends Rql2TruffleCompilerTestContext { private val allTypes = tempFile(""" | 120 @@ -527,7 +528,7 @@ trait XmlPackageTest extends Rql2CompilerTestContext { test(snapi"""Xml.Read("$junkAfter10Items", type collection(record(a: int, b: string, c: double)))""")( _ should runErrorAs( - snapi"failed to read XML (line 10 column 49) (url: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" + snapi"failed to read XML (line 10 column 49) (location: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" ) ) @@ -551,7 +552,7 @@ trait XmlPackageTest extends Rql2CompilerTestContext { snapi"""Collection.Take(Xml.Read("$junkAfter10Items", type collection(record(a: int, b: string, c: double))), 11)""" )( _ should runErrorAs( - snapi"failed to read XML (line 10 column 49) (url: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" + snapi"failed to read XML (line 10 column 49) (location: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" ) ) @@ -559,7 +560,7 @@ trait XmlPackageTest extends Rql2CompilerTestContext { snapi"""Collection.Count(Xml.Read("$junkAfter10Items", type collection(record(a: int, b: string, c: double))))""".stripMargin )( _ should runErrorAs( - snapi"failed to read XML (line 10 column 49) (url: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" + snapi"failed to read XML (line 10 column 49) (location: $junkAfter10Items): Unexpected character '#' (code 35) in epilog" ) ) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionDistinctTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionDistinctTest.scala index 6348f1a3f..d193a5125 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionDistinctTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionDistinctTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait CollectionDistinctTest extends Rql2CompilerTestContext { +@TruffleTests class CollectionDistinctTest extends Rql2TruffleCompilerTestContext { test("""let numbers = Collection.From([5, 2, 4, 2, 2, 4, 5]) |in Collection.Distinct(numbers)""".stripMargin)(_ should evaluateTo("[2, 4, 5]")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionExplodeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionExplodeTest.scala index b92212f92..dd294a415 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionExplodeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionExplodeTest.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait CollectionExplodeTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class CollectionExplodeTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val nba = tempFile("""[ | {"team": {"name": "Bulls", "city": "Chicago"}, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionGroupByTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionGroupByTest.scala index 5dbce4bc3..3af9a9c25 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionGroupByTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionGroupByTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ import raw.compiler.rql2.errors.KeyNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait CollectionGroupByTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class CollectionGroupByTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val lineitemType = """record( | l_orderkey: int, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionJoinTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionJoinTest.scala index a3ffabbf6..763f02def 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionJoinTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionJoinTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ import raw.compiler.rql2.errors.KeyNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait CollectionJoinTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class CollectionJoinTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test(snapi"""let regions = Csv.InferAndRead("$tpchRegionCsvLocal"), | nations = Csv.InferAndRead("$tpchNationCsvLocal") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMinMaxTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMinMaxTest.scala index 875798f33..cb8fa6fa1 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMinMaxTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMinMaxTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait CollectionMinMaxTest extends Rql2CompilerTestContext { +@TruffleTests class CollectionMinMaxTest extends Rql2TruffleCompilerTestContext { // Date test(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMkStringTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMkStringTest.scala index a9e3d11fb..4e8c3f1b3 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMkStringTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionMkStringTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin.collection -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait CollectionMkStringTest extends Rql2CompilerTestContext { +@TruffleTests class CollectionMkStringTest extends Rql2TruffleCompilerTestContext { test("""let items = Collection.Build() |in Collection.MkString(items, sep="|")""".stripMargin)(_ should evaluateTo(""" "" """)) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionOrderByTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionOrderByTest.scala index 7036d17d7..bc496369a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionOrderByTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionOrderByTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ import raw.compiler.rql2.errors.{InvalidOrderSpec, KeyNotComparable, OrderSpecMustFollowOrderingFunction} -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait CollectionOrderByTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class CollectionOrderByTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val coolBandsWithoutHeader = tempFile("""Bee Gees|Maurice|Gibb|1949 |Bee Gees|Robin|Gibb|1949 diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionPackageTest.scala index 28202f693..772cb803d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionPackageTest.scala @@ -14,11 +14,12 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait CollectionPackageTest extends Rql2CompilerTestContext { +@TruffleTests class CollectionPackageTest extends Rql2TruffleCompilerTestContext { // a test to check if there are multiple instances of generators test("""let a = Collection.Build(1,2,3), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionRangeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionRangeTest.scala index 08a4b6bb3..10f1713bd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionRangeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionRangeTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin.collection -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait CollectionRangeTest extends Rql2CompilerTestContext { +@TruffleTests class CollectionRangeTest extends Rql2TruffleCompilerTestContext { // Long.Range diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionUnionTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionUnionTest.scala index 309c86141..7c7647484 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionUnionTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/collection/CollectionUnionTest.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.builtin.collection import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait CollectionUnionTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class CollectionUnionTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test("Collection.Union(Collection.Build(1,2,3), Collection.Build(4,5,6))")(_ should evaluateTo("[1,2,3,4,5,6]")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/AwsPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/AwsPackageTest.scala index ee36f6f38..62286b4d1 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/AwsPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/AwsPackageTest.scala @@ -12,18 +12,16 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext { +@TruffleTests class AwsPackageTest extends Rql2TruffleCompilerTestContext { val accessKeyId = sys.env("RAW_AWS_ACCESS_KEY_ID") val secretAccessKey = sys.env("RAW_AWS_SECRET_ACCESS_KEY") - secret(authorizedUser, "aws-secret-key", secretAccessKey) - secret(authorizedUser, "aws-access-key", accessKeyId) - - def xmlImplemented: Boolean + secret("aws-secret-key", secretAccessKey) + secret("aws-access-key", accessKeyId) val triple = "\"\"\"" @@ -41,7 +39,6 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |) |in List.Filter(data.regionInfo.item, x -> x.regionName == "us-east-1") |""".stripMargin) { it => - assume(xmlImplemented) it should evaluateTo("""[ {regionName: "us-east-1", regionEndpoint: "ec2.us-east-1.amazonaws.com"} ]""") } @@ -61,7 +58,6 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |) |in List.Filter(data.regionInfo.item, x -> x.regionName == "us-east-1") |""".stripMargin) { it => - assume(xmlImplemented) it should evaluateTo("""[ {regionName: "us-east-1", regionEndpoint: "ec2.us-east-1.amazonaws.com"} ]""") } @@ -82,7 +78,6 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext | ) |in List.Filter(data.regionInfo.item, x -> x.regionName == "eu-west-1") |""".stripMargin) { it => - assume(xmlImplemented) it should evaluateTo("""[ {regionName: "eu-west-1", regionEndpoint: "ec2.eu-west-1.amazonaws.com"} ]""") } @@ -99,7 +94,6 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |) |in Collection.Filter(data.regionInfo.item, x -> x.regionName == "us-east-1") |""".stripMargin) { it => - assume(xmlImplemented) it should evaluateTo("""[ {regionName: "us-east-1", regionEndpoint: "ec2.us-east-1.amazonaws.com"} ]""") } @@ -179,7 +173,6 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |in Collection.Count(data.reservationSet.item) | |""".stripMargin) { it => - assume(xmlImplemented) // it should run it should evaluateTo("""0""") } @@ -211,10 +204,7 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |) |in List.Filter(data.ListUsersResult.Users.member, x -> x.UserName == "dummy-user").UserName | - |""".stripMargin) { it => - assume(xmlImplemented) - it should evaluateTo(""" [] """) - } + |""".stripMargin)(it => it should evaluateTo(""" [] """)) // querying monitoring aka could-watch to get cpu usage // this test was failing with Non-terminating decimal expansion; no exact representable decimal result. @@ -270,7 +260,7 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext | ] | ) |) - |""".stripMargin)(it => it should runErrorAs("http error: host not found for https://iam.us-east-1.amazonaws.com/")) + |""".stripMargin)(it => it should runErrorAs("host not found for https://iam.us-east-1.amazonaws.com/")) // Wrong path. test("""String.Read( @@ -304,7 +294,7 @@ trait AwsPackageTest extends Rql2CompilerTestContext with CredentialsTestContext |) | |""".stripMargin)( - _ should runErrorAs("http error: host not found for https://does-not-exist.amazonaws.com/") + _ should runErrorAs("host not found for https://does-not-exist.amazonaws.com/") ) // Wrong service but with correct host. diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/EnvironmentPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/EnvironmentPackageTest.scala index e21b0841b..67b716112 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/EnvironmentPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/EnvironmentPackageTest.scala @@ -12,20 +12,20 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait EnvironmentPackageTest extends Rql2CompilerTestContext with CredentialsTestContext { +@TruffleTests class EnvironmentPackageTest extends Rql2TruffleCompilerTestContext { - secret(authorizedUser, "my-secret", "my-secret-value") + secret("my-secret", "my-secret-value") test("""Environment.Secret("my-secret")""")(it => it should evaluateTo(""" "my-secret-value" """)) - test("""Environment.Secret("my-typo")""")(it => it should runErrorAs("could not find secret my-typo")) + test("""Environment.Secret("my-typo")""")(it => it should runErrorAs("unknown secret: my-typo")) // checking it doesn't fail if one of the secrets is not found (RD-9041) test("""[Environment.Secret("my-secret"), Environment.Secret("my-typo")]""")(it => - it should evaluateTo("""["my-secret-value", Error.Build("could not find secret my-typo")] + it should evaluateTo("""["my-secret-value", Error.Build("unknown secret: my-typo")] |""".stripMargin) ) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/LocationPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/LocationPackageTest.scala index 712a1de60..fbeba57bd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/LocationPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/LocationPackageTest.scala @@ -12,38 +12,35 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.creds.dropbox.DropboxTestCreds -import raw.creds.s3.S3TestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LocationPackageTest - extends Rql2CompilerTestContext - with CredentialsTestContext - with DropboxTestCreds - with S3TestCreds { +@TruffleTests class LocationPackageTest extends Rql2TruffleCompilerTestContext { - oauth(authorizedUser, "dropbox-refresh-token", dropboxRefreshTokenCredential) + import raw.compiler.rql2.tests.TestCredentials._ + + httpHeaders("dropbox-token", Map("Authorization" -> ("Bearer " + dropboxLongLivedAccessToken))) + + s3Bucket(UnitTestPrivateBucket2, UnitTestPrivateBucket2Cred) property("raw.sources.dropbox.clientId", dropboxClientId) test(""" |String.Read( - | Location.Build( + | Http.Post( | "https://api.dropboxapi.com/2/users/get_space_usage", - | http_method = "POST", - | http_auth_cred_name = "dropbox-refresh-token" + | authCredentialName = "dropbox-token" | ) |)""".stripMargin)(it => it should run) // reading a public s3 bucket without registering or passing credentials test(s"""let - | data = Csv.InferAndRead("s3://${UnitTestPublicBucket.name}/students.csv") + | data = Csv.InferAndRead("s3://$UnitTestPublicBucket/students.csv") |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) - test(s"""Location.Ls("s3://${UnitTestPublicBucket.name}/publications/")""") { it => + test(s"""Location.Ls("s3://$UnitTestPublicBucket/publications/")""") { it => it should evaluateTo("""Collection.Build( | "s3://rawlabs-public-test-data/publications/authors.parquet", | "s3://rawlabs-public-test-data/publications/authors.hjson", @@ -61,7 +58,7 @@ trait LocationPackageTest |""".stripMargin) } - test(s"""Location.Ll("s3://${UnitTestPublicBucket.name}/publications/authors.hjson") + test(s"""Location.Ll("s3://$UnitTestPublicBucket/publications/authors.hjson") |""".stripMargin) { it => it should evaluateTo("""Collection.Build( | Record.Build( @@ -79,7 +76,7 @@ trait LocationPackageTest // Cannot count lists test(s""" |let - | data = Location.Ll("s3://${UnitTestPublicBucket.name}/publications") + | data = Location.Ll("s3://$UnitTestPublicBucket/publications") |in | List.Count(data) |""".stripMargin)(it => it should evaluateTo("""12L""".stripMargin)) @@ -87,21 +84,19 @@ trait LocationPackageTest // reading a non public s3 bucket passing credentials in the location settings test(s"""let | data = Csv.InferAndRead( - | Location.Build( - | "s3://${UnitTestPrivateBucket.name}/students.csv", - | s3_region = "${UnitTestPrivateBucket.region.get}", - | s3_access_key = "${UnitTestPrivateBucket.credentials.get.accessKey}", - | s3_secret_key = "${UnitTestPrivateBucket.credentials.get.secretKey}" + | S3.Build( + | "s3://$UnitTestPrivateBucket/students.csv", + | region = "${UnitTestPrivateBucketCred.region.get}", + | accessKey = "${UnitTestPrivateBucketCred.accessKey.get}", + | secretKey = "${UnitTestPrivateBucketCred.secretKey.get}" | ) | ) |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) - s3Bucket(authorizedUser, UnitTestPrivateBucket2) - // using a private bucket registered in the credentials server - test(s"""String.Read(Location.Build("s3://${UnitTestPrivateBucket2.name}/file1.csv")) + test(s"""String.Read(S3.Build("s3://$UnitTestPrivateBucket2/file1.csv")) |""".stripMargin)(it => it should evaluateTo(""" "foobar" """)) } diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/MySQLPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/MySQLPackageTest.scala index d1c42c1cc..05e5a5d65 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/MySQLPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/MySQLPackageTest.scala @@ -12,16 +12,18 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class MySQLPackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ val mysqlRegDb = "registered-db" val mysqlTable = "tbl1" - rdbms(authorizedUser, mysqlRegDb, mysqlCreds) + rdbms(mysqlRegDb, mysqlCreds) private val ttt = "\"\"\"" // Trying all types. Not all expressions type as wanted so that @@ -112,7 +114,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte test( s"""MySQL.InferAndRead("${mysqlCreds.database}", "$mysqlTable", - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}")""".stripMargin + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}")""".stripMargin ) { it => it should evaluateTo( """[ @@ -126,7 +128,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte test( s"""MySQL.Read("${mysqlCreds.database}", "$mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}")""".stripMargin + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}")""".stripMargin ) { it => it should orderEvaluateTo( """[ @@ -137,13 +139,13 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte ) } - test(s""" + ignore(s""" |let - | d = Location.Describe(Location.Build( + | d = Location.Describe(MySQL.Build( | "mysql://${mysqlCreds.database}/$mysqlTable", - | db_host = "${mysqlCreds.host}", - | db_username = "${mysqlCreds.username.get.toString}", - | db_password = "${mysqlCreds.password.get.toString}" + | host = "${mysqlCreds.host}", + | username = "${mysqlCreds.username}", + | password = "${mysqlCreds.password}" | )) |in | d.columns @@ -159,32 +161,30 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte // no credentials test( s"""MySQL.InferAndRead("${mysqlCreds.database}", "$mysqlTable" )""".stripMargin - )(it => - it should runErrorAs(s"""inference error: no credential found for mysql: ${mysqlCreds.database}""".stripMargin) - ) + )(it => it should runErrorAs(s"""unknown database credential: ${mysqlCreds.database}""".stripMargin)) test( s"""MySQL.Read("${mysqlCreds.database}", "$mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)) |)""".stripMargin - )(it => it should runErrorAs(s"""no credential found for mysql: ${mysqlCreds.database}""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: ${mysqlCreds.database}""".stripMargin)) // server does not exist test( s"""MySQL.Read( | "${badMysqlCreds.database}", "$mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${badMysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}" + | host = "${badMysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}" |)""".stripMargin )(it => it should runErrorAs(s"""unknown host: ${badMysqlCreds.host}""".stripMargin)) // wrong port - // When there is a wrong port supplied the test takes a long time to run and we get an connect time out error. + // Note that when there is a wrong port supplied, the test takes a long time to run and we get a connect time out error. ignore( s"""MySQL.Read( | "${mysqlCreds.database}", "$mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}", port = 1234 + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}", port = 1234 |)""".stripMargin )(it => it should runErrorAs(s"""connect timed out: ${mysqlCreds.database}""".stripMargin)) @@ -195,14 +195,14 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), | host = "${mysqlCreds.host}" |)""".stripMargin - )(it => it should runErrorAs("""authentication failed""".stripMargin)) + )(it => it should runErrorAs("""username is required""".stripMargin)) // wrong password test( s"""MySQL.Read( | "${mysqlCreds.database}", "$mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "wrong!" + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "wrong!" |)""".stripMargin )(it => it should runErrorAs("""authentication failed""".stripMargin)) @@ -218,7 +218,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte test( s"""MySQL.InferAndQuery("${mysqlCreds.database}", "SELECT * FROM $mysqlTable", - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}" )""".stripMargin + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -244,7 +244,7 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte test( s"""MySQL.Query("${mysqlCreds.database}", "SELECT * FROM $mysqlTable", | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get.toString}", password = "${mysqlCreds.password.get.toString}" )""".stripMargin + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", password = "${mysqlCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -297,8 +297,8 @@ trait MySQLPackageTest extends Rql2CompilerTestContext with CredentialsTestConte | Collection.Count( | MySQL.Query("${mysqlCreds.database}", "SELECT * FROM " + table, | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${mysqlCreds.host}", username = "${mysqlCreds.username.get}", - | password = "${mysqlCreds.password.get}") + | host = "${mysqlCreds.host}", username = "${mysqlCreds.username}", + | password = "${mysqlCreds.password}") | ))""".stripMargin ) { it => val error = diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/OraclePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/OraclePackageTest.scala index ed8eeb02d..944e20e03 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/OraclePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/OraclePackageTest.scala @@ -12,17 +12,19 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class OraclePackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ val oracleDb = "rawdb" val oracleSchema = "rawtest" val oracleTable = "tbl1" - rdbms(authorizedUser, "oracle", oracleCreds) + rdbms("oracle", oracleCreds) test(s"""Oracle.InferAndRead("oracle", "$oracleSchema", "$oracleTable")""") { it => assume(!compilerService.language.contains("rql2-truffle")) @@ -66,7 +68,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont test( s"""Oracle.InferAndRead("$oracleDb", "$oracleSchema", "$oracleTable", - | host = "${oracleCreds.host}", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}")""".stripMargin + | host = "${oracleCreds.host}", username = "${oracleCreds.username}", password = "${oracleCreds.password}")""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) it should evaluateTo( @@ -81,7 +83,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont test( s"""Oracle.Read("$oracleDb", "$oracleSchema", "$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: int, Y: string)), - | host = "${oracleCreds.host}", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}" )""".stripMargin + | host = "${oracleCreds.host}", username = "${oracleCreds.username}", password = "${oracleCreds.password}" )""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) it should orderEvaluateTo( @@ -93,13 +95,13 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont ) } - test(s""" + ignore(s""" |let - | d = Location.Describe(Location.Build( + | d = Location.Describe(Oracle.Build( | "oracle://$oracleDb/$oracleSchema/$oracleTable", - | db_host = "${oracleCreds.host}", - | db_username = "${oracleCreds.username.get.toString}", - | db_password = "${oracleCreds.password.get.toString}" + | host = "${oracleCreds.host}", + | username = "${oracleCreds.username}", + | password = "${oracleCreds.password}" | )) |in | d.columns @@ -115,7 +117,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont // no credentials test( s"""Oracle.InferAndRead("$oracleDb", "$oracleSchema", "$oracleTable" )""".stripMargin - )(it => it should runErrorAs(s"""inference error: no credential found for oracle: $oracleDb""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: $oracleDb""".stripMargin)) test( s"""Oracle.Read("$oracleSchema", "rdbmstest", "$oracleTable", @@ -131,7 +133,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont s"""Oracle.Read( | "$oracleDb", "$oracleSchema", "$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: int, Y: string)), - | host = "oracle.localdomain", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}" + | host = "oracle.localdomain", username = "${oracleCreds.username}", password = "${oracleCreds.password}" |)""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) @@ -143,7 +145,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont s"""Oracle.Read( | "$oracleDb", "$oracleSchema", "$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: int, Y: string)), - | host = "localhost", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}" + | host = "localhost", username = "${oracleCreds.username}", password = "${oracleCreds.password}" |)""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) @@ -156,7 +158,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont s"""Oracle.Read( | "$oracleDb", "$oracleSchema", "$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: int, Y: string)), - | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}", port = 1234 + | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username}", password = "${oracleCreds.password}", port = 1234 |)""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) @@ -172,7 +174,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont |)""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) - it should runErrorAs("""authentication failed""".stripMargin) + it should runErrorAs("""username is required""".stripMargin) } // wrong password @@ -180,7 +182,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont s"""Oracle.Read( | "$oracleDb", "$oracleSchema", "$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: int, Y: string)), - | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username.get.toString}", password = "wrong!" + | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username}", password = "wrong!" |)""".stripMargin ) { it => assume(!compilerService.language.contains("rql2-truffle")) @@ -199,7 +201,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont test( s"""Oracle.InferAndQuery("$oracleDb", "SELECT * FROM $oracleSchema.$oracleTable", - | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}" )""".stripMargin + | host = "test-oracle.raw-labs.com", username = "${oracleCreds.username}", password = "${oracleCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -225,7 +227,7 @@ trait OraclePackageTest extends Rql2CompilerTestContext with CredentialsTestCont test( s"""Oracle.Query("$oracleDb", "SELECT * FROM $oracleSchema.$oracleTable", | type collection(record(A: int, B: int, C: double, D: double, X: string, Y: string)), - | host = "${oracleCreds.host}", username = "${oracleCreds.username.get.toString}", password = "${oracleCreds.password.get.toString}" )""".stripMargin + | host = "${oracleCreds.host}", username = "${oracleCreds.username}", password = "${oracleCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/PostgreSQLPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/PostgreSQLPackageTest.scala index 8b43b90af..a965f96b1 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/PostgreSQLPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/PostgreSQLPackageTest.scala @@ -12,16 +12,18 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class PostgreSQLPackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ val pgSchema = "rdbmstest" val pgTable = "tbl1" - rdbms(authorizedUser, "pgsql", pgsqlCreds) + rdbms("pgsql", pgsqlCreds) private val ttt = "\"\"\"" // Trying all types. Not all expressions type as wanted so that @@ -112,7 +114,7 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest test( s"""PostgreSQL.InferAndRead("${pgsqlCreds.database}", "$pgSchema", "$pgTable", - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}")""".stripMargin + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}")""".stripMargin ) { it => it should evaluateTo( """[ @@ -126,7 +128,7 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest test( s"""PostgreSQL.Read("${pgsqlCreds.database}", "$pgSchema", "$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}" )""".stripMargin + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}" )""".stripMargin ) { it => it should orderEvaluateTo( """[ @@ -137,13 +139,13 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest ) } - test(s""" + ignore(s""" |let - | d = Location.Describe(Location.Build( + | d = Location.Describe(PostgreSQL.Build( | "pgsql://${pgsqlCreds.database}/$pgSchema/$pgTable", - | db_host = "${pgsqlCreds.host}", - | db_username = "${pgsqlCreds.username.get.toString}", - | db_password = "${pgsqlCreds.password.get.toString}" + | host = "${pgsqlCreds.host}", + | username = "${pgsqlCreds.username}", + | password = "${pgsqlCreds.password}" | )) |in | d.columns @@ -159,22 +161,20 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest // no credentials test( s"""PostgreSQL.InferAndRead("${pgsqlCreds.database}", "$pgSchema", "$pgTable" )""".stripMargin - ) { it => - it should runErrorAs(s"""inference error: no credential found for postgresql: ${pgsqlCreds.database}""".stripMargin) - } + )(it => it should runErrorAs(s"""unknown database credential: ${pgsqlCreds.database}""".stripMargin)) test( s"""PostgreSQL.Read("${pgsqlCreds.database}", "$pgSchema", "$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)) |)""".stripMargin - )(it => it should runErrorAs(s"""no credential found for postgresql: ${pgsqlCreds.database}""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: ${pgsqlCreds.database}""".stripMargin)) // server does not exist test( s"""PostgreSQL.Read( | "${pgsqlCreds.database}", "$pgSchema", "$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${badMysqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}" + | host = "${badMysqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}" |)""".stripMargin )(it => it should runErrorAs(s"""unknown host: ${badMysqlCreds.host}""".stripMargin)) @@ -184,7 +184,7 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest s"""PostgreSQL.Read( | "${pgsqlCreds.database}", "$pgSchema", "$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}", port = 1234 + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}", port = 1234 |)""".stripMargin )(it => it should runErrorAs(s"""connect timed out: ${pgsqlCreds.host}""".stripMargin)) @@ -195,16 +195,16 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), | host = "${pgsqlCreds.host}" |)""".stripMargin - )(it => it should runErrorAs(s"""error connecting to database: ${pgsqlCreds.host}""".stripMargin)) + )(it => it should runErrorAs("username is required")) // wrong password test( s"""PostgreSQL.Read( | "${pgsqlCreds.database}", "$pgSchema", "$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "wrong!" + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "wrong!" |)""".stripMargin - )(it => it should runErrorAs("""authentication failed""".stripMargin)) + )(it => it should runErrorAs("authentication failed")) test(s"""PostgreSQL.InferAndQuery("pgsql", "SELECT * FROM $pgSchema.$pgTable")""") { it => it should evaluateTo( @@ -218,7 +218,7 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest test( s"""PostgreSQL.InferAndQuery("${pgsqlCreds.database}", "SELECT * FROM $pgSchema.$pgTable", - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}" )""".stripMargin + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -244,7 +244,7 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest test( s"""PostgreSQL.Query("${pgsqlCreds.database}", "SELECT * FROM $pgSchema.$pgTable", | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get.toString}", password = "${pgsqlCreds.password.get.toString}" )""".stripMargin + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", password = "${pgsqlCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -261,8 +261,8 @@ trait PostgreSQLPackageTest extends Rql2CompilerTestContext with CredentialsTest | Collection.Count( | PostgreSQL.Query("${pgsqlCreds.database}", "SELECT * FROM $pgSchema." + table, | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username.get}", - | password = "${pgsqlCreds.password.get}") + | host = "${pgsqlCreds.host}", username = "${pgsqlCreds.username}", + | password = "${pgsqlCreds.password}") | ))""".stripMargin ) { it => val error = diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/S3PackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/S3PackageTest.scala index 40d17d8f2..0af280d78 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/S3PackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/S3PackageTest.scala @@ -12,40 +12,42 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.creds.s3.S3TestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait S3PackageTest extends Rql2CompilerTestContext with CredentialsTestContext with S3TestCreds { +@TruffleTests class S3PackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ + + s3Bucket(UnitTestPrivateBucket2, UnitTestPrivateBucket2Cred) // reading a non public s3 bucket passing credentials in the location settings test(s"""let | data = Csv.InferAndRead( | S3.Build( | "s3://rawlabs-private-test-data/students.csv", - | region = "${UnitTestPrivateBucket.region.get}", - | accessKey = "${UnitTestPrivateBucket.credentials.get.accessKey}", - | secretKey = "${UnitTestPrivateBucket.credentials.get.secretKey}" + | region = "${UnitTestPrivateBucketCred.region.get}", + | accessKey = "${UnitTestPrivateBucketCred.accessKey.get}", + | secretKey = "${UnitTestPrivateBucketCred.secretKey.get}" | ) | ) |in | Collection.Count(data) |""".stripMargin)(it => it should evaluateTo("7")) - s3Bucket(authorizedUser, UnitTestPrivateBucket2) - // using a private bucket registered in the credentials server - test(s"""String.Read(S3.Build("s3://${UnitTestPrivateBucket2.name}/file1.csv")) + test(s"""String.Read(S3.Build("s3://$UnitTestPrivateBucket2/file1.csv")) |""".stripMargin)(it => it should evaluateTo(""" "foobar" """)) // listing a s3 bucket from us-east-1 (non default region) test(s"""let | data = Location.Ls( | S3.Build( - | "s3://${unitTestPrivateBucketUsEast1.name}/csvs/01", - | region = "${unitTestPrivateBucketUsEast1.region.get}", - | accessKey = "${unitTestPrivateBucketUsEast1.credentials.get.accessKey}", - | secretKey = "${unitTestPrivateBucketUsEast1.credentials.get.secretKey}" + | "s3://$unitTestPrivateBucketUsEast1/csvs/01", + | region = "${unitTestPrivateBucketUsEast1Cred.region.get}", + | accessKey = "${unitTestPrivateBucketUsEast1Cred.accessKey.get}", + | secretKey = "${unitTestPrivateBucketUsEast1Cred.secretKey.get}" | ) | ) |in @@ -59,9 +61,9 @@ trait S3PackageTest extends Rql2CompilerTestContext with CredentialsTestContext test(s"""let | data = Location.Ls( | S3.Build( - | "s3://${unitTestPrivateBucketUsEast1.name}/csvs/01", - | accessKey = "${unitTestPrivateBucketUsEast1.credentials.get.accessKey}", - | secretKey = "${unitTestPrivateBucketUsEast1.credentials.get.secretKey}" + | "s3://$unitTestPrivateBucketUsEast1/csvs/01", + | accessKey = "${unitTestPrivateBucketUsEast1Cred.accessKey.get}", + | secretKey = "${unitTestPrivateBucketUsEast1Cred.secretKey.get}" | ) | ) |in diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SqlServerPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SQLServerPackageTest.scala similarity index 86% rename from snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SqlServerPackageTest.scala rename to snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SQLServerPackageTest.scala index 6db6ff6c7..c6ccff467 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SqlServerPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SQLServerPackageTest.scala @@ -12,18 +12,20 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class SQLServerPackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ val sqlServDb = "rawtest" val sqlServRegDb = "registered-db" val sqlServSchema = "rdbmstest" val sqlServTable = "tbl1" - rdbms(authorizedUser, sqlServRegDb, sqlServerCreds) + rdbms(sqlServRegDb, sqlServerCreds) private val ttt = "\"\"\"" test(s"""SQLServer.InferAndQuery("$sqlServRegDb", $ttt @@ -110,7 +112,7 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""SQLServer.InferAndRead("$sqlServDb", "$sqlServSchema", "$sqlServTable", - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}")""".stripMargin + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}")""".stripMargin ) { it => it should evaluateTo( """[ @@ -124,7 +126,7 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""SQLServer.Read("$sqlServDb", "$sqlServSchema", "$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}" )""".stripMargin + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}" )""".stripMargin ) { it => it should orderEvaluateTo( """[ @@ -135,13 +137,13 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC ) } - test(s""" + ignore(s""" |let - | d = Location.Describe(Location.Build( + | d = Location.Describe(SQLServer.Build( | "sqlserver://$sqlServDb/$sqlServSchema/$sqlServTable", - | db_host = "${sqlServerCreds.host}", - | db_username = "${sqlServerCreds.username.get.toString}", - | db_password = "${sqlServerCreds.password.get.toString}" + | host = "${sqlServerCreds.host}", + | username = "${sqlServerCreds.username}", + | password = "${sqlServerCreds.password}" | )) |in | d.columns @@ -157,20 +159,20 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC // no credentials test( s"""SQLServer.InferAndRead("$sqlServDb", "$sqlServSchema", "$sqlServTable" )""".stripMargin - )(it => it should runErrorAs(s"""inference error: no credential found for sqlserver: rawtest""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: rawtest""".stripMargin)) test( s"""SQLServer.Read("$sqlServDb", "$sqlServSchema", "$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)) |)""".stripMargin - )(it => it should runErrorAs(s"""no credential found for sqlserver: $sqlServDb""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: $sqlServDb""".stripMargin)) // server does not exist test( s"""SQLServer.Read( | "$sqlServDb", "$sqlServSchema", "$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "does-not-exist", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}" + | host = "does-not-exist", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}" |)""".stripMargin )(it => it should runErrorAs("""error connecting to database: does-not-exist""".stripMargin)) @@ -180,7 +182,7 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC s"""SQLServer.Read( | "$sqlServDb", "$sqlServSchema", "$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}", port = 1234 + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}", port = 1234 |)""".stripMargin )(it => it should runErrorAs(s"""connect timed out: ${sqlServerCreds.host}""".stripMargin)) @@ -191,16 +193,16 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), | host = "${sqlServerCreds.host}" |)""".stripMargin - )(it => it should runErrorAs("""Login failed for user ''""".stripMargin)) + )(it => it should runErrorAs("username is required")) // wrong password test( s"""SQLServer.Read( | "$sqlServDb", "$sqlServSchema", "$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "wrong!" + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "wrong!" |)""".stripMargin - )(it => it should runErrorAs(s"""Login failed for user '${sqlServerCreds.username.get.toString}'""".stripMargin)) + )(it => it should runErrorAs(s"""Login failed for user '${sqlServerCreds.username}'""".stripMargin)) test(s"""SQLServer.InferAndQuery("$sqlServRegDb", "SELECT * FROM $sqlServSchema.$sqlServTable")""") { it => it should evaluateTo( @@ -214,7 +216,7 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""SQLServer.InferAndQuery("$sqlServDb", "SELECT * FROM $sqlServSchema.$sqlServTable", - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}" )""".stripMargin + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -240,7 +242,7 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""SQLServer.Query("$sqlServDb", "SELECT * FROM $sqlServSchema.$sqlServTable", | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get.toString}", password = "${sqlServerCreds.password.get.toString}" )""".stripMargin + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", password = "${sqlServerCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -257,8 +259,8 @@ trait SqlServerPackageTest extends Rql2CompilerTestContext with CredentialsTestC | Collection.Count( | SQLServer.Query("$sqlServDb", "SELECT * FROM $sqlServSchema." + table, | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username.get}", - | password = "${sqlServerCreds.password.get}") + | host = "${sqlServerCreds.host}", username = "${sqlServerCreds.username}", + | password = "${sqlServerCreds.password}") | ))""".stripMargin ) { it => val error = diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SnowflakePackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SnowflakePackageTest.scala index 44ce94fe6..b3bb66a96 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SnowflakePackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/credentials/SnowflakePackageTest.scala @@ -12,18 +12,19 @@ package raw.compiler.rql2.tests.builtin.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds -import raw.sources.jdbc.snowflake.SnowflakeClient +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class SnowflakePackageTest extends Rql2TruffleCompilerTestContext { + + import TestCredentials._ val snowflakeSchema = "PUBLIC" val snowflakeMainTable = "TBL1" val snowflakeSideTable = "TBL4" - rdbms(authorizedUser, "snowflake", snowflakeCreds) + rdbms("snowflake", snowflakeCreds) private val ttt = "\"\"\"" // Trying all types. Not all expressions type as wanted so that @@ -75,14 +76,6 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC |}]""".stripMargin) } - override def beforeAll(): Unit = { - super.beforeAll() - // (CTM) while we sort out our snowflake installation - // this will fail immediately instead of polluting tier2 tests - val client = new SnowflakeClient(snowflakeCreds)(settings) - client.listSchemas - } - property("raw.sources.rdbms.network-timeout", "10s") test(s"""Snowflake.InferAndRead("snowflake", "$snowflakeSchema", "$snowflakeMainTable")""") { it => @@ -224,7 +217,7 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.InferAndRead("${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable", - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get.toString}", password = "${snowflakeCreds.password.get.toString}")""".stripMargin + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", password = "${snowflakeCreds.password}")""".stripMargin ) { it => it should evaluateTo( """[ @@ -238,7 +231,7 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.Read("${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get.toString}", password = "${snowflakeCreds.password.get.toString}" )""".stripMargin + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", password = "${snowflakeCreds.password}" )""".stripMargin ) { it => it should orderEvaluateTo( """[ @@ -249,13 +242,13 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC ) } - test(s""" + ignore(s""" |let - | d = Location.Describe(Location.Build( + | d = Location.Describe(Snowflake.Build( | "snowflake://${snowflakeCreds.database}/$snowflakeSchema/$snowflakeMainTable", - | db_account_id = "${snowflakeCreds.accountIdentifier}", - | db_username = "${snowflakeCreds.username.get.toString}", - | db_password = "${snowflakeCreds.password.get.toString}" + | accountID = "${snowflakeCreds.accountIdentifier}", + | username = "${snowflakeCreds.username}", + | password = "${snowflakeCreds.password}" | )) |in | d.columns @@ -273,7 +266,7 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC s"""Snowflake.InferAndRead("${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable" )""".stripMargin )(it => it should runErrorAs( - s"""inference error: no credential found for Snowflake: ${snowflakeCreds.database}""".stripMargin + s"""unknown database credential: ${snowflakeCreds.database}""".stripMargin ) ) @@ -281,18 +274,18 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC s"""Snowflake.Read("${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)) |)""".stripMargin - )(it => it should runErrorAs(s"""no credential found for Snowflake: ${snowflakeCreds.database}""".stripMargin)) + )(it => it should runErrorAs(s"""unknown database credential: ${snowflakeCreds.database}""".stripMargin)) // server does not exist test( s"""Snowflake.Read( | "${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | accountID = "does-not-exist", username = "${snowflakeCreds.username.get.toString}", password = "${snowflakeCreds.password.get.toString}" + | accountID = "does-not-exist", username = "${snowflakeCreds.username}", password = "${snowflakeCreds.password}" |)""".stripMargin ) { it => it should runErrorAs( - """IO error connecting to does-not-exist: JDBC driver encountered communication error. Message: HTTP status=403.""".stripMargin + """IO error connecting to does-not-exist: JDBC driver encountered communication error. Message: HTTP status=513.""".stripMargin ) } @@ -303,14 +296,14 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), | accountID = "${snowflakeCreds.accountIdentifier}" |)""".stripMargin - )(it => it should runErrorAs(s"""authentication failed""".stripMargin)) + )(it => it should runErrorAs(s"""username is required""".stripMargin)) // wrong password test( s"""Snowflake.Read( | "${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeMainTable", | type collection(record(a: int, b: int, c: double, d: double, x: int, y: string)), - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get.toString}", password = "wrong!" + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", password = "wrong!" |)""".stripMargin ) { it => it should runErrorAs( @@ -330,7 +323,7 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.InferAndQuery("${snowflakeCreds.database}", "SELECT * FROM public.$snowflakeMainTable", - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get.toString}", password = "${snowflakeCreds.password.get.toString}" )""".stripMargin + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", password = "${snowflakeCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -356,7 +349,7 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.Query("${snowflakeCreds.database}", "SELECT * FROM public.$snowflakeMainTable", | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get.toString}", password = "${snowflakeCreds.password.get.toString}" )""".stripMargin + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", password = "${snowflakeCreds.password}" )""".stripMargin ) { it => it should evaluateTo( """[ @@ -370,8 +363,8 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.InferAndRead("${snowflakeCreds.database}", "$snowflakeSchema", "$snowflakeSideTable", | accountID = "${snowflakeCreds.accountIdentifier}", - | username = "${snowflakeCreds.username.get.toString}", - | password = "${snowflakeCreds.password.get.toString}", + | username = "${snowflakeCreds.username}", + | password = "${snowflakeCreds.password}", | options = [{"timezone", "America/Los_Angeles"}] |)""".stripMargin ) { it => @@ -414,8 +407,8 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC test( s"""Snowflake.InferAndQuery("${snowflakeCreds.database}", "SELECT * FROM $snowflakeSchema.$snowflakeSideTable", | accountID = "${snowflakeCreds.accountIdentifier}", - | username = "${snowflakeCreds.username.get.toString}", - | password = "${snowflakeCreds.password.get.toString}", + | username = "${snowflakeCreds.username}", + | password = "${snowflakeCreds.password}", | options = [{"timezone", "UTC"}] |)""".stripMargin ) { it => @@ -461,8 +454,8 @@ trait SnowflakePackageTest extends Rql2CompilerTestContext with CredentialsTestC | Collection.Count( | Snowflake.Query("${snowflakeCreds.database}", "SELECT * FROM public." + table, | type collection(record(a: int, b: int, c: double, d: double, x: string, y: string)), - | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username.get}", - | password = "${snowflakeCreds.password.get}")))""".stripMargin + | accountID = "${snowflakeCreds.accountIdentifier}", username = "${snowflakeCreds.username}", + | password = "${snowflakeCreds.password}")))""".stripMargin ) { it => val error = s"""failed to read from database snowflake:${snowflakeCreds.database}: SQL compilation error:\\nObject '${snowflakeCreds.database.toUpperCase}.PUBLIC.DONT_EXIST' does not exist or not authorized.""".stripMargin diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListDistinctTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListDistinctTest.scala index 5428bcb80..e725ca5b5 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListDistinctTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListDistinctTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ListDistinctTest extends Rql2CompilerTestContext { +@TruffleTests class ListDistinctTest extends Rql2TruffleCompilerTestContext { test("""let numbers = [5, 2, 4, 2, 2, 4, 5] |in List.Distinct(numbers)""".stripMargin)(_ should evaluateTo("[2, 4, 5]")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListExplodeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListExplodeTest.scala index a0ebef0cb..352d7a690 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListExplodeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListExplodeTest.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait ListExplodeTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class ListExplodeTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val nba = tempFile("""[ | {"team": {"name": "Bulls", "city": "Chicago"}, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListGroupByTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListGroupByTest.scala index 00c4b0218..470a6439f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListGroupByTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListGroupByTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ import raw.compiler.rql2.errors.KeyNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait ListGroupByTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class ListGroupByTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val lineitemType = """record( | l_orderkey: int, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListJoinTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListJoinTest.scala index d3448ebbe..5928ac0ee 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListJoinTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListJoinTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ import raw.compiler.rql2.errors.KeyNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait ListJoinTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class ListJoinTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { private val listOfCountries = """[ | {region: "AFRICA", nation: "ALGERIA"}, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMinMaxTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMinMaxTest.scala index 0577336a4..fddc0bf26 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMinMaxTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMinMaxTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ListMinMaxTest extends Rql2CompilerTestContext { +@TruffleTests class ListMinMaxTest extends Rql2TruffleCompilerTestContext { // Date test(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMkStringTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMkStringTest.scala index 87186e314..c78a3431c 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMkStringTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListMkStringTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.builtin.list -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ListMkStringTest extends Rql2CompilerTestContext { +@TruffleTests class ListMkStringTest extends Rql2TruffleCompilerTestContext { test("""let items = List.Build() |in List.MkString(items, sep="|")""".stripMargin)(_ should evaluateTo(""" "" """)) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListOrderByTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListOrderByTest.scala index 31043e819..e6c64b32d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListOrderByTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListOrderByTest.scala @@ -14,10 +14,11 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ import raw.compiler.rql2.errors.{InvalidOrderSpec, KeyNotComparable} -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait ListOrderByTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class ListOrderByTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { // using String.ReadLines test(snapi"""let regions = List.From(Collection.Transform( diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListPackageTest.scala index ade63ff4f..66358aa79 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListPackageTest.scala @@ -14,11 +14,12 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ import raw.compiler.rql2.errors.ItemsNotComparable -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait ListPackageTest extends Rql2CompilerTestContext { +@TruffleTests class ListPackageTest extends Rql2TruffleCompilerTestContext { test("""["Hello", Error.Build("Argh!!"), null]""") { _ should evaluateTo("""["Hello", Error.Build("Argh!!"), null]""") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListUnionTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListUnionTest.scala index 4037aa442..e6e560b47 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListUnionTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/builtin/list/ListUnionTest.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.builtin.list import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait ListUnionTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class ListUnionTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test("List.Union([1,2,3], [4,5,6])")(_ should evaluateTo("[1,2,3,4,5,6]")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/hints/SemanticAnalyzerHintsTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/hints/SemanticAnalyzerHintsTest.scala index e2e8a95b7..d6f271e28 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/hints/SemanticAnalyzerHintsTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/hints/SemanticAnalyzerHintsTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.hints -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait SemanticAnalyzerHintsTest extends Rql2CompilerTestContext { +@TruffleTests class SemanticAnalyzerHintsTest extends Rql2TruffleCompilerTestContext { test("""Rekord.Build(a=1)""")(it => it should typeErrorAs("did you mean Record.Build?")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspAiValidateTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspAiValidateTest.scala index 2ba1c8f18..f9dc0e2b0 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspAiValidateTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspAiValidateTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api.ValidateResponse import raw.compiler.rql2.errors.OutputTypeRequiredForRecursiveFunction -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspAiValidateTest extends Rql2CompilerTestContext { +@TruffleTests class LspAiValidateTest extends Rql2TruffleCompilerTestContext { test("AI validate package that does not exist") { _ => val code = """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspBrokenCodeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspBrokenCodeTest.scala index 1749225bc..278ea73db 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspBrokenCodeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspBrokenCodeTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspBrokenCodeTest extends Rql2CompilerTestContext { +@TruffleTests class LspBrokenCodeTest extends Rql2TruffleCompilerTestContext { test("broken code hover test") { _ => val code = """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCommentsFormatTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCommentsFormatTest.scala index 3a217e568..68de5229e 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCommentsFormatTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCommentsFormatTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.lsp -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspCommentsFormatTest extends Rql2CompilerTestContext { +@TruffleTests class LspCommentsFormatTest extends Rql2TruffleCompilerTestContext { def assertFormattedCode( code: String, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCompilationMessagesTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCompilationMessagesTest.scala index 0dd117257..01a226201 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCompilationMessagesTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspCompilationMessagesTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api.{ErrorMessage, WarningMessage} import raw.compiler.base.errors.{MissingSecretWarning, UnknownDecl} -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspCompilationMessagesTest extends Rql2CompilerTestContext { +@TruffleTests class LspCompilationMessagesTest extends Rql2TruffleCompilerTestContext { test("should return a waning") { _ => val code = """let a = Environment.Secret("a") in a""".stripMargin diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDefinitionTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDefinitionTest.scala index 03c713f31..422eb949f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDefinitionTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDefinitionTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.lsp -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspDefinitionTest extends Rql2CompilerTestContext { +@TruffleTests class LspDefinitionTest extends Rql2TruffleCompilerTestContext { test("go to definition identifier at usage test") { _ => val code = """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDotAutoCompleteTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDotAutoCompleteTest.scala index 8727740af..fb271b685 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDotAutoCompleteTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspDotAutoCompleteTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspDotAutoCompleteTest extends Rql2CompilerTestContext { +@TruffleTests class LspDotAutoCompleteTest extends Rql2TruffleCompilerTestContext { private def dotAutoCompleteTest(code: String, line: Int, col: Int, expectedFields: Seq[(String, String)]): Unit = { val AutoCompleteResponse(entries) = dotAutoComplete(code, Pos(line, col)) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspFormatCodeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspFormatCodeTest.scala index 95e0944cb..b1f9d2c31 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspFormatCodeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspFormatCodeTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api.FormatCodeResponse -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspFormatCodeTest extends Rql2CompilerTestContext { +@TruffleTests class LspFormatCodeTest extends Rql2TruffleCompilerTestContext { def assertFormattedCode(code: String, expected: String) = { val FormatCodeResponse(Some(formattedCode)) = formatCode(code) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspHoverTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspHoverTest.scala index d67676dd8..59453cf7f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspHoverTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspHoverTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.lsp import raw.client.api._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspHoverTest extends Rql2CompilerTestContext { +@TruffleTests class LspHoverTest extends Rql2TruffleCompilerTestContext { test("hover identifier at definition test") { _ => val code = """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspRenameTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspRenameTest.scala index 85b9e4289..093afa981 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspRenameTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspRenameTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.lsp -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspRenameTest extends Rql2CompilerTestContext { +@TruffleTests class LspRenameTest extends Rql2TruffleCompilerTestContext { test("rename identifier at usage test") { _ => val code = """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspValidateTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspValidateTest.scala index 2895428d6..4aa937cd6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspValidateTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspValidateTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.lsp -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspValidateTest extends Rql2CompilerTestContext { +@TruffleTests class LspValidateTest extends Rql2TruffleCompilerTestContext { test("validate simple code test") { _ => val code = """String.Lower("Hello")""".stripMargin diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspWordAutoCompleteTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspWordAutoCompleteTest.scala index 11a46bab0..244321cfb 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspWordAutoCompleteTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/lsp/LspWordAutoCompleteTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.lsp -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LspWordAutoCompleteTest extends Rql2CompilerTestContext { +@TruffleTests class LspWordAutoCompleteTest extends Rql2TruffleCompilerTestContext { private lazy val allTypes = Seq( ("byte", Some("")), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/KryoPackageTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/KryoPackageTest.scala similarity index 93% rename from snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/KryoPackageTest.scala rename to snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/KryoPackageTest.scala index 8166879a6..aaafaa67d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/KryoPackageTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/KryoPackageTest.scala @@ -10,14 +10,14 @@ * licenses/APL.txt. */ -package raw.compiler.rql2.truffle.offheap +package raw.compiler.rql2.tests.offheap import org.scalatest.prop.TableDrivenPropertyChecks -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.testing.tags.TruffleTests @TruffleTests -class KryoPackageTest extends TruffleWithLocalCredentialsTestContext with TableDrivenPropertyChecks { +class KryoPackageTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { private val cases = Table( ("value", "type"), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDatasets.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDatasets.scala index 7028fc3ea..8bbcbed40 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDatasets.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDatasets.scala @@ -13,9 +13,9 @@ package raw.compiler.rql2.tests.offheap import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext -trait OffHeapDatasets extends Rql2CompilerTestContext { +trait OffHeapDatasets { this: Rql2TruffleCompilerTestContext => protected val N = 100 diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDistinctTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDistinctTest.scala index 9472e438f..6ce4803a3 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDistinctTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapDistinctTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.offheap import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait OffHeapDistinctTest extends Rql2CompilerTestContext { +@TruffleTests class OffHeapDistinctTest extends Rql2TruffleCompilerTestContext { // This is to ensure the test triggers spill to disk. property("raw.runtime.external.disk-block-max-size", "30kB") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapEquiJoinTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapEquiJoinTest.scala index cb096af6b..a305897ea 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapEquiJoinTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapEquiJoinTest.scala @@ -12,9 +12,11 @@ package raw.compiler.rql2.tests.offheap +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait OffHeapEquiJoinTest extends OffHeapDatasets { +@TruffleTests class OffHeapEquiJoinTest extends Rql2TruffleCompilerTestContext with OffHeapDatasets { // This is to ensure the test triggers spill to disk. property("raw.runtime.external.disk-block-max-size", "20kB") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapGroupTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapGroupTest.scala index a4a91318f..088a02628 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapGroupTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapGroupTest.scala @@ -12,9 +12,11 @@ package raw.compiler.rql2.tests.offheap +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait OffHeapGroupTest extends OffHeapDatasets { +@TruffleTests class OffHeapGroupTest extends Rql2TruffleCompilerTestContext with OffHeapDatasets { // This is to ensure the test triggers spill to disk. property("raw.runtime.external.disk-block-max-size", "20kB") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapJoinTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapJoinTest.scala index d0d45ef98..8320face6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapJoinTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapJoinTest.scala @@ -12,9 +12,11 @@ package raw.compiler.rql2.tests.offheap +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait OffHeapJoinTest extends OffHeapDatasets { +@TruffleTests class OffHeapJoinTest extends Rql2TruffleCompilerTestContext with OffHeapDatasets { // This is to ensure the test triggers spill to disk. property("raw.runtime.external.disk-block-max-size", "20kB") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapOrderByTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapOrderByTest.scala index c9e2b3fde..a3f9df4c8 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapOrderByTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/offheap/OffHeapOrderByTest.scala @@ -12,9 +12,11 @@ package raw.compiler.rql2.tests.offheap +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait OffHeapOrderByTest extends OffHeapDatasets { +@TruffleTests class OffHeapOrderByTest extends Rql2TruffleCompilerTestContext with OffHeapDatasets { // This is to ensure the test triggers spill to disk. property("raw.runtime.external.disk-block-max-size", "20kB") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/BinaryOutputTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/BinaryOutputTest.scala index ef26ba568..0cce5ff50 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/BinaryOutputTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/BinaryOutputTest.scala @@ -13,13 +13,14 @@ package raw.compiler.rql2.tests.output import org.apache.commons.io.FileUtils -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests import java.io.File import java.nio.file.Files -trait BinaryOutputTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class BinaryOutputTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { option("output-format", "binary") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala index f889dbc8f..de6f22c7d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala @@ -12,13 +12,14 @@ package raw.compiler.rql2.tests.output -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests import raw.utils._ import java.nio.file.Files -trait CsvOutputTest extends Rql2CompilerTestContext { +@TruffleTests class CsvOutputTest extends Rql2TruffleCompilerTestContext { option("output-format", "csv") @@ -121,7 +122,7 @@ trait CsvOutputTest extends Rql2CompilerTestContext { if (compilerService.language.contains("rql2-truffle")) { path should contain( snapi"""byteCol,shortCol,intCol,longCol,floatCol,doubleCol,decimalCol,boolCol,nullBoolCol,dateCol,timeCol,timestampCol - |"failed to parse CSV (url: $csvWithAllTypes: line 1, col 1), cannot parse 'byteCol' as a byte","failed to parse CSV (url: $csvWithAllTypes: line 1, col 9), cannot parse 'shortCol' as a short","failed to parse CSV (url: $csvWithAllTypes: line 1, col 18), cannot parse 'intCol' as an int","failed to parse CSV (url: $csvWithAllTypes: line 1, col 25), cannot parse 'longCol' as a long","failed to parse CSV (url: $csvWithAllTypes: line 1, col 33), cannot parse 'floatCol' as a float","failed to parse CSV (url: $csvWithAllTypes: line 1, col 42), cannot parse 'doubleCol' as a double","failed to parse CSV (url: $csvWithAllTypes: line 1, col 52), cannot parse 'decimalCol' as a decimal","failed to parse CSV (url: $csvWithAllTypes: line 1, col 63), cannot parse 'boolCol' as a bool","failed to parse CSV (url: $csvWithAllTypes: line 1, col 71), cannot parse 'nullBoolCol' as a bool","failed to parse CSV (url: $csvWithAllTypes: line 1, col 83), string 'dateCol' does not match date template 'yyyy-M-d'","failed to parse CSV (url: $csvWithAllTypes: line 1, col 91), string 'timeCol' does not match time template 'HH:mm[:ss[.SSS]]'","failed to parse CSV (url: $csvWithAllTypes: line 1, col 99), string 'timestampCol' does not match timestamp template 'HH:mm[:ss[.SSS]]'" + |"failed to parse CSV (location: $csvWithAllTypes: line 1, col 1), cannot parse 'byteCol' as a byte","failed to parse CSV (location: $csvWithAllTypes: line 1, col 9), cannot parse 'shortCol' as a short","failed to parse CSV (location: $csvWithAllTypes: line 1, col 18), cannot parse 'intCol' as an int","failed to parse CSV (location: $csvWithAllTypes: line 1, col 25), cannot parse 'longCol' as a long","failed to parse CSV (location: $csvWithAllTypes: line 1, col 33), cannot parse 'floatCol' as a float","failed to parse CSV (location: $csvWithAllTypes: line 1, col 42), cannot parse 'doubleCol' as a double","failed to parse CSV (location: $csvWithAllTypes: line 1, col 52), cannot parse 'decimalCol' as a decimal","failed to parse CSV (location: $csvWithAllTypes: line 1, col 63), cannot parse 'boolCol' as a bool","failed to parse CSV (location: $csvWithAllTypes: line 1, col 71), cannot parse 'nullBoolCol' as a bool","failed to parse CSV (location: $csvWithAllTypes: line 1, col 83), string 'dateCol' does not match date template 'yyyy-M-d'","failed to parse CSV (location: $csvWithAllTypes: line 1, col 91), string 'timeCol' does not match time template 'HH:mm[:ss[.SSS]]'","failed to parse CSV (location: $csvWithAllTypes: line 1, col 99), string 'timestampCol' does not match timestamp template 'HH:mm[:ss[.SSS]]'" |1,10,100,1000,3.14,6.28,9.42,true,false,2023-12-25,01:02:03,2023-12-25T01:02:03 |120,2500,25000,9223372036854775807,30.14,60.28,90.42,false,,2023-02-05,11:12:13,2023-02-05T11:12:13 |""".stripMargin diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/JsonOutputTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/JsonOutputTest.scala index 2db9aa0cb..b2cca0c8c 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/JsonOutputTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/JsonOutputTest.scala @@ -12,12 +12,13 @@ package raw.compiler.rql2.tests.output -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import raw.utils._ import java.nio.file.{Files, Path} -trait JsonOutputTest extends Rql2CompilerTestContext { +@TruffleTests class JsonOutputTest extends Rql2TruffleCompilerTestContext { option("output-format", "json") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/TextOutputTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/TextOutputTest.scala index 8718391ee..cf10324a1 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/TextOutputTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/output/TextOutputTest.scala @@ -12,11 +12,12 @@ package raw.compiler.rql2.tests.output -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Files -trait TextOutputTest extends Rql2CompilerTestContext { +@TruffleTests class TextOutputTest extends Rql2TruffleCompilerTestContext { option("output-format", "text") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/FrontendSyntaxAnalyzerTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/FrontendSyntaxAnalyzerTest.scala index 30d7924e6..473d5d11d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/FrontendSyntaxAnalyzerTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/FrontendSyntaxAnalyzerTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.parser -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait FrontendSyntaxAnalyzerTest extends Rql2CompilerTestContext { +@TruffleTests class FrontendSyntaxAnalyzerTest extends Rql2TruffleCompilerTestContext { // Internal node, not visible to the user parser. test(s"""$$package("Collection")""".stripMargin)(it => it shouldNot parse) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/ListSugarTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/ListSugarTest.scala index 79159adb8..b9b7f5e03 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/ListSugarTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/ListSugarTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.parser -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ListSugarTest extends Rql2CompilerTestContext { +@TruffleTests class ListSugarTest extends Rql2TruffleCompilerTestContext { test("""[1,2,3]""") { it => it should typeAs("list(int)") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/OperatorPrecedenceTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/OperatorPrecedenceTest.scala index 334a4025f..311612b19 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/OperatorPrecedenceTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/OperatorPrecedenceTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.parser import raw.compiler.common.source._ import raw.compiler.rql2.source._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait OperatorPrecedenceTest extends Rql2CompilerTestContext { +@TruffleTests class OperatorPrecedenceTest extends Rql2TruffleCompilerTestContext { test("""1+2*3""") { it => it should evaluateTo("(1+(2*3))") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/RecordSugarTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/RecordSugarTest.scala index 962aad7b4..aabc50e5f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/RecordSugarTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/parser/RecordSugarTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.parser -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RecordSugarTest extends Rql2CompilerTestContext { +@TruffleTests class RecordSugarTest extends Rql2TruffleCompilerTestContext { test("""{a: 1, b: "2"}""") { it => it should typeAs("record(a: int, b: string)") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10194Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10194Test.scala index 4227947ab..6a6784e67 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10194Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10194Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD10194Test extends Rql2CompilerTestContext { +@TruffleTests class RD10194Test extends Rql2TruffleCompilerTestContext { private val q = """main(title: string = null, description: string = null, category: string = null, rating: string = null, actor: string = null) = @@ -26,22 +27,7 @@ trait RD10194Test extends Rql2CompilerTestContext { | rating_clause = if(Nullable.IsNull(rating)) then "" else " and rating like \"%"+rating+"%\"", | actor_clause = if(Nullable.IsNull(actor)) then "" else " and actors like \"%"+actor+"%\"", | where_clause = " where true " + actor_clause + title_clause + description_clause + category_clause + rating_clause, - | full_query = "select * from nicer_but_slower_film_list "+where_clause, - | hello = MySQL.Query( - | "sakila", - | full_query, - | type collection( - | record( - | FID: short, - | title: string, - | description: string, - | category: string, - | price: decimal, - | length: short, - | rating: string, - | actors: string, - | ) - | )) + | full_query = "select * from nicer_but_slower_film_list "+where_clause | in | full_query | diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10220Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10220Test.scala index 34b9a1dda..8f96a89cb 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10220Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10220Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD10220Test extends Rql2CompilerTestContext { +@TruffleTests class RD10220Test extends Rql2TruffleCompilerTestContext { test("""Csv.InferAndParse("stringData")""")( _ should (typeAs("collection(record(stringData: undefined))") and evaluateTo("[]")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10723Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10723Test.scala index 95ccf5b64..b96901d0e 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10723Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10723Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD10723Test extends Rql2CompilerTestContext { +@TruffleTests class RD10723Test extends Rql2TruffleCompilerTestContext { private val qqq = "\"\"\"" test(s"""main() = diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/RD10767Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10767Test.scala similarity index 73% rename from snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/RD10767Test.scala rename to snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10767Test.scala index 1c93d282a..ebe95e7c7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/RD10767Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10767Test.scala @@ -10,13 +10,13 @@ * licenses/APL.txt. */ -package raw.compiler.rql2.truffle.truffle.regressions +package raw.compiler.rql2.tests.regressions import raw.client.api.{GetProgramDescriptionSuccess, ProgramEnvironment} -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils.SnapiInterpolator -class RD10767Test extends TruffleWithLocalCredentialsTestContext { +class RD10767Test extends Rql2TruffleCompilerTestContext { private val data = tempFile(""" |[ | {"a": 1, "b": 10, "c": 100}, @@ -28,7 +28,18 @@ class RD10767Test extends TruffleWithLocalCredentialsTestContext { | |Json.Read("$data", data_type()) |""".stripMargin) { it => - val programEnvironment = ProgramEnvironment(authorizedUser, None, Set.empty, Map.empty, None) + val programEnvironment = ProgramEnvironment( + authorizedUser, + None, + Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + None, + None + ) compilerService.getProgramDescription(it.q, programEnvironment) match { case GetProgramDescriptionSuccess(desc) => assert(desc.maybeRunnable.isDefined, "Expected a runnable program") @@ -42,7 +53,18 @@ class RD10767Test extends TruffleWithLocalCredentialsTestContext { | |func()(1) |""".stripMargin) { it => - val programEnvironment = ProgramEnvironment(authorizedUser, None, Set.empty, Map.empty, None) + val programEnvironment = ProgramEnvironment( + authorizedUser, + None, + Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + None, + None + ) compilerService.getProgramDescription(it.q, programEnvironment) match { case GetProgramDescriptionSuccess(desc) => assert(desc.maybeRunnable.isDefined, "Expected a runnable program") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10801Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10801Test.scala index e2b92ad53..8c12e029d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10801Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD10801Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils.SnapiInterpolator +import raw.testing.tags.TruffleTests -trait RD10801Test extends Rql2CompilerTestContext { +@TruffleTests class RD10801Test extends Rql2TruffleCompilerTestContext { private val qqq = "\"\"\"" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3742Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3742Test.scala index 9d98e0802..ca083642a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3742Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3742Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD3742Test extends Rql2CompilerTestContext { +@TruffleTests class RD3742Test extends Rql2TruffleCompilerTestContext { val triple = "\"\"\"" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3784Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3784Test.scala index d03f6fa9c..fdcb143fd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3784Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD3784Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD3784Test extends Rql2CompilerTestContext { +@TruffleTests class RD3784Test extends Rql2TruffleCompilerTestContext { private val data = tempFile( """v diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4529Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4529Test.scala index 0ceb9addc..34873cf5a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4529Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4529Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD4529Test extends Rql2CompilerTestContext { +@TruffleTests class RD4529Test extends Rql2TruffleCompilerTestContext { test(""" |let a: float = 1.0f diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4981Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4981Test.scala index 93431322c..8ba2a798f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4981Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD4981Test.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait RD4981Test extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class RD4981Test extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test(snapi"""let | data = Csv.InferAndRead("$airportsLocal"), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5238Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5238Test.scala index a091d7f1d..8a7112424 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5238Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5238Test.scala @@ -13,10 +13,11 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait RD5238Test extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class RD5238Test extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test(snapi""" |let region1 = Csv.InferAndRead("$tpchRegionCsvLocal"), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5365Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5365Test.scala index 379c05f74..90beee77a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5365Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5365Test.scala @@ -17,9 +17,10 @@ import org.scalatest.matchers.{MatchResult, Matcher} import raw.utils.TestData import raw.compiler.rql2.FrontendSyntaxAnalyzer import raw.compiler.rql2.source.SourcePrettyPrinter -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5365Test extends Rql2CompilerTestContext { +@TruffleTests class RD5365Test extends Rql2TruffleCompilerTestContext { private class PrettyPrintAs(expected: String) extends Matcher[TestData] { override def apply(query: TestData): MatchResult = { diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5393Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5393Test.scala index 1f8890da4..60cdf9ebd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5393Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5393Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.rql2.errors.CannotDetermineTypeOfParameter -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5393Test extends Rql2CompilerTestContext { +@TruffleTests class RD5393Test extends Rql2TruffleCompilerTestContext { test("""main(country: string = null,code: string = null) = | let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5412Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5412Test.scala index b48f49a1c..b22aaf53c 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5412Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5412Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5412Test extends Rql2CompilerTestContext { +@TruffleTests class RD5412Test extends Rql2TruffleCompilerTestContext { test("""let | json_type = type record(creation_date: string, entries: list(string)), @@ -41,6 +42,6 @@ trait RD5412Test extends Rql2CompilerTestContext { | List.Union( | read_logs("s3://bucketA/*.json", awsAccountA), | read_logs("s3://bucketB/*.json", awsAccountB))""".stripMargin)( - _ should runErrorAs("could not find secret") + _ should runErrorAs("unknown secret") ) } diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5448Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5448Test.scala index 27a4296bd..d0757e364 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5448Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5448Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5448Test extends Rql2CompilerTestContext { +@TruffleTests class RD5448Test extends Rql2TruffleCompilerTestContext { private val ttt = "\"\"\"" private val jsonString = """[ diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5484Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5484Test.scala index 976b5d44b..a83bc5d7a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5484Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5484Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5484Test extends Rql2CompilerTestContext { +@TruffleTests class RD5484Test extends Rql2TruffleCompilerTestContext { // top-level and correct. test("""Timestamp.Build(2015, 1, 4, 22, 0)""")(_ should run) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5488Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5488Test.scala index d8bf5bea9..0b0d5736a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5488Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5488Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5488Test extends Rql2CompilerTestContext { +@TruffleTests class RD5488Test extends Rql2TruffleCompilerTestContext { // All these tests are valid queries. They fail in the middle of compilation, at L0, with "lines is declared more than once". diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5491Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5491Test.scala index e83e757f4..668b5cc50 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5491Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5491Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5491Test extends Rql2CompilerTestContext { +@TruffleTests class RD5491Test extends Rql2TruffleCompilerTestContext { test(s"""let | query = \"\"\"SELECT (?item as ?cat) ?itemLabel diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5644Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5644Test.scala index 8f55a8a24..7db042578 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5644Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5644Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5644Test extends Rql2CompilerTestContext { +@TruffleTests class RD5644Test extends Rql2TruffleCompilerTestContext { // Basic reference behavior. The function parameter isn't type and inherits the list item type. Everything matches. test(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5679Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5679Test.scala index 27c9f4305..d50c818a6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5679Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5679Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5679Test extends Rql2CompilerTestContext { +@TruffleTests class RD5679Test extends Rql2TruffleCompilerTestContext { // regular XML file (types as record) with a list inside private val xml = tempFile(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5685Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5685Test.scala index a1446da76..9c4680be7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5685Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5685Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5685Test extends Rql2CompilerTestContext { +@TruffleTests class RD5685Test extends Rql2TruffleCompilerTestContext { test("""let itemType = type int, | listType = type list(itemType), diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5691Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5691Test.scala index 9d1c2e99e..cbb1e4502 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5691Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5691Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext +import raw.testing.tags.TruffleTests -trait RD5691Test extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class RD5691Test extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { test(s""" |let issueType = type record( diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5697Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5697Test.scala index 0f4fab7e2..699d375cc 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5697Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5697Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait RD5697Test extends Rql2CompilerTestContext { +@TruffleTests class RD5697Test extends Rql2TruffleCompilerTestContext { private val jsonFile = tempFile("""{"a": 12, "b": 14}""") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5714Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5714Test.scala index 6cdb71c0b..aadbcdfc4 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5714Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5714Test.scala @@ -13,11 +13,12 @@ package raw.compiler.rql2.tests.regressions import java.nio.file.Files -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import scala.io.Source -trait RD5714Test extends Rql2CompilerTestContext { +@TruffleTests class RD5714Test extends Rql2TruffleCompilerTestContext { test("""let colA = [{id: 1, name: "john"}], | colB = [{id: 2, firstName: "john"}], diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5722Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5722Test.scala index 750a95e0a..b753605db 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5722Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5722Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5722Test extends Rql2CompilerTestContext { +@TruffleTests class RD5722Test extends Rql2TruffleCompilerTestContext { test("""let data = Collection.Build({ | a: { b: 123} diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD572Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD572Test.scala index 1fd14300a..4ce0ead02 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD572Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD572Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD572Test extends Rql2CompilerTestContext { +@TruffleTests class RD572Test extends Rql2TruffleCompilerTestContext { test("-128b") { it => it should typeAs("byte") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5775Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5775Test.scala index 1ba29616a..a543943b5 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5775Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5775Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5775Test extends Rql2CompilerTestContext { +@TruffleTests class RD5775Test extends Rql2TruffleCompilerTestContext { test("""let data = Http.Get("https://jira.atlassian.com/rest/api/latest/search", args=[{"jql", "fixVersion=9.0.0"}]), | r = Json.InferAndRead(data), | issues = Collection.Transform(r.issues, i -> { diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5779Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5779Test.scala index 0715d7c7f..05ac9db29 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5779Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5779Test.scala @@ -13,11 +13,12 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait RD5779Test extends Rql2CompilerTestContext { +@TruffleTests class RD5779Test extends Rql2TruffleCompilerTestContext { val data: Path = tempFile("""[ | {"a": 1, "b": 10} |]""".stripMargin) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5784Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5784Test.scala index 683456db3..598a5a7f6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5784Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5784Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait RD5784Test extends Rql2CompilerTestContext { +@TruffleTests class RD5784Test extends Rql2TruffleCompilerTestContext { private val xmlFile = tempFile(""" | diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5785Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5785Test.scala index 421b72216..7404e19b5 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5785Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5785Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5785Test extends Rql2CompilerTestContext { +@TruffleTests class RD5785Test extends Rql2TruffleCompilerTestContext { // The only way of this returning true is if all values in the list are the same. // Almost impossible with 5 random values. diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5786Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5786Test.scala index 785b711f9..5de62e286 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5786Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5786Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5786Test extends Rql2CompilerTestContext { +@TruffleTests class RD5786Test extends Rql2TruffleCompilerTestContext { test("""true == false""") { _ should evaluateTo(""" false """) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5797Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5797Test.scala deleted file mode 100644 index 67f4423e1..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5797Test.scala +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.tests.regressions - -import raw.compiler.rql2.tests.Rql2CompilerTestContext - -trait RD5797Test extends Rql2CompilerTestContext { - - // "raw.eu.auth0.com" is a wrong value, it should be 'auth0' for example - test(""" - |let - | hello = String.Read(Http.Get("https://...", - | clientId="...", - | clientSecret="..-..-..", - | authProvider="raw.eu.auth0.com")) - |in - | hello""".stripMargin)(_ should runErrorAs("invalid credential type")) - - test(""" - |let - | hello = Json.InferAndRead(Http.Get("https://...", - | clientId="...", - | clientSecret="..-..-..", - | authProvider="raw.eu.auth0.com")) - |in - | hello""".stripMargin)(_ should runErrorAs("invalid credential type")) - - test(""" - |let - | hello = Json.Read(Http.Get("https://...", - | clientId="...", - | clientSecret="..-..-..", - | authProvider="raw.eu.auth0.com"), type list(int)) - |in - | hello""".stripMargin)(_ should runErrorAs("invalid credential type")) - - test(""" - |let - | hello = Collection.Sum(Json.Read(Http.Get("https://...", - | clientId="...", - | clientSecret="..-..-..", - | authProvider="raw.eu.auth0.com"), type collection(int))) - |in - | hello""".stripMargin)(_ should runErrorAs("invalid credential type")) -} diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5851Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5851Test.scala index 8d2621edf..7cc8ac528 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5851Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5851Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5851Test extends Rql2CompilerTestContext { +@TruffleTests class RD5851Test extends Rql2TruffleCompilerTestContext { def autoCompleteNames(entries: Array[Completion]): Seq[String] = { entries.map { diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5884Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5884Test.scala index fa4dbbe45..2bd494147 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5884Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5884Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5884Test extends Rql2CompilerTestContext { +@TruffleTests class RD5884Test extends Rql2TruffleCompilerTestContext { test("""let f(x: int, y: int = 12) = 3.14 + x + y |in f(1)""".stripMargin)(_ should run) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5893Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5893Test.scala index 59365d455..8fe399622 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5893Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5893Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5893Test extends Rql2CompilerTestContext { +@TruffleTests class RD5893Test extends Rql2TruffleCompilerTestContext { // this has a list of text private val data = tempFile(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5914Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5914Test.scala index 0c2a6ec42..891850718 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5914Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5914Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5914Test extends Rql2CompilerTestContext { +@TruffleTests class RD5914Test extends Rql2TruffleCompilerTestContext { test("""let item1 = {name: "coffee machine", price: 200, price: 199}, // price is duplicated, price is an int | item2 = {name: "coffee machine", price: 200.00, price: 199.99} // price is duplicated, price is a double diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5920Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5920Test.scala index f38b846d7..5cd93fb87 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5920Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5920Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5920Test extends Rql2CompilerTestContext { +@TruffleTests class RD5920Test extends Rql2TruffleCompilerTestContext { test("""let f(x: int) = x * 10 |in f(10)""".stripMargin)(_ should evaluateTo("100")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5921Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5921Test.scala index cffe87623..4539281ae 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5921Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5921Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5921Test extends Rql2CompilerTestContext { +@TruffleTests class RD5921Test extends Rql2TruffleCompilerTestContext { val data1 = tempFile("""1 |2 diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5925Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5925Test.scala index e306507d0..5c7d24a08 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5925Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5925Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5925Test extends Rql2CompilerTestContext { +@TruffleTests class RD5925Test extends Rql2TruffleCompilerTestContext { test("""let c = Collection.Build(1, 2, 3, 4) |in Collection.Filter(c, s -> null)""".stripMargin) { it => diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5932Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5932Test.scala index 60a018485..774410959 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5932Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5932Test.scala @@ -13,10 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.creds.s3.S3TestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5932Test extends Rql2CompilerTestContext with S3TestCreds { +@TruffleTests class RD5932Test extends Rql2TruffleCompilerTestContext { val data = tempFile("""[ | {"id": 1, "network_interface": "eni-08b85cc07294f82bf"}, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5968Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5968Test.scala index 810e4200f..3acc73efd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5968Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5968Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5968Test extends Rql2CompilerTestContext { +@TruffleTests class RD5968Test extends Rql2TruffleCompilerTestContext { private val cdataFile = tempFile(""" | diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5971Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5971Test.scala index 64c20a661..65153b606 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5971Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5971Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5971Test extends Rql2CompilerTestContext { +@TruffleTests class RD5971Test extends Rql2TruffleCompilerTestContext { // error when type isn't supported test(s"""[{a: "binary", b: [1,2,3,4]}]""") { it => diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5979Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5979Test.scala index 49be2fd3b..d4b514d3f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5979Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD5979Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5979Test extends Rql2CompilerTestContext { +@TruffleTests class RD5979Test extends Rql2TruffleCompilerTestContext { test("""[{a: 12, b: 14}, {c: 23, d: 54}]""")(_ should runErrorAs("expected compatible with")) test("""[{a: 12, b: 14}, {a: 23, d: 54}]""")(_ should runErrorAs("expected compatible with")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7924Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7924Test.scala index db9a30da5..b7a9bfd4a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7924Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7924Test.scala @@ -13,11 +13,12 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Path -trait RD7924Test extends Rql2CompilerTestContext { +@TruffleTests class RD7924Test extends Rql2TruffleCompilerTestContext { val string: Path = tempFile(""" "Hello!" """) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7974Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7974Test.scala index f2fe43cdc..6a6967a3c 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7974Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD7974Test.scala @@ -13,12 +13,13 @@ package raw.compiler.rql2.tests.regressions import org.scalatest.BeforeAndAfterEach -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import raw.utils._ import java.nio.file.Files -trait RD7974Test extends Rql2CompilerTestContext with BeforeAndAfterEach { +@TruffleTests class RD7974Test extends Rql2TruffleCompilerTestContext with BeforeAndAfterEach { private val tmpFile = Files.createTempFile("csv-output-test", ".csv") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8530Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8530Test.scala index 5723dabac..087c1aa9a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8530Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8530Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import org.scalatest.BeforeAndAfterEach -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD8530Test extends Rql2CompilerTestContext with BeforeAndAfterEach { +@TruffleTests class RD8530Test extends Rql2TruffleCompilerTestContext with BeforeAndAfterEach { test("""List.Filter([1,2,3,4], n -> n > 2, a = 12)""".stripMargin)( _ should typeErrorAs("no optional arguments expected") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8764Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8764Test.scala index b237e9f0b..31271b42e 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8764Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8764Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import com.typesafe.scalalogging.StrictLogging -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD8764Test extends Rql2CompilerTestContext with StrictLogging { +@TruffleTests class RD8764Test extends Rql2TruffleCompilerTestContext with StrictLogging { test(""" |let f: (record(_1: int, _2: int)) -> bool = (x: int, y: int) -> x == y diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8935Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8935Test.scala index ae84f26b9..a928b1976 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8935Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8935Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD8935Test extends Rql2CompilerTestContext { +@TruffleTests class RD8935Test extends Rql2TruffleCompilerTestContext { private val vowels = tempFile("""a |e diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8993Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8993Test.scala index 6e5590fac..136961c17 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8993Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD8993Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD8993Test extends Rql2CompilerTestContext { +@TruffleTests class RD8993Test extends Rql2TruffleCompilerTestContext { test("""apply(f: int -> bool) = f(1) |apply((x: int) -> true)""".stripMargin)(_ should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9137Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9137Test.scala index 0431cddec..73f9bd406 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9137Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9137Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import com.typesafe.scalalogging.StrictLogging -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9137Test extends Rql2CompilerTestContext with StrictLogging { +@TruffleTests class RD9137Test extends Rql2TruffleCompilerTestContext with StrictLogging { test("""Json.InferAndRead("https://raw-tutorial.s3.eu-west-1.amazonaws.com/patients.json")""".stripMargin)( _ should run diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9228Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9228Test.scala index 97cc45be0..a6a22cd16 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9228Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9228Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9228Test extends Rql2CompilerTestContext { +@TruffleTests class RD9228Test extends Rql2TruffleCompilerTestContext { // pass a plain URL. It will be turned into a location, directly passed as a parameter. test(""" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9229Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9229Test.scala index a6bfc0026..081b2fe43 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9229Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9229Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9229Test extends Rql2CompilerTestContext { +@TruffleTests class RD9229Test extends Rql2TruffleCompilerTestContext { test(s"""[{a: "binary", b: [1,2,3,4]}]""") { it => option("output-format", "binary") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9255Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9255Test.scala index 3cde06f71..b82c61f9e 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9255Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9255Test.scala @@ -13,12 +13,13 @@ package raw.compiler.rql2.tests.regressions import org.scalatest.EitherValues -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.time.{Duration, LocalDate, LocalDateTime, LocalTime} -trait RD9255Test extends Rql2CompilerTestContext with EitherValues { +@TruffleTests class RD9255Test extends Rql2TruffleCompilerTestContext with EitherValues { private val declarations = """ |string_func(x: string) = String.Length(x) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9359Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9359Test.scala index 6b44ba45d..767eedb5d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9359Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9359Test.scala @@ -12,12 +12,13 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests import java.nio.file.{Files, Path} -trait RD9359Test extends Rql2CompilerTestContext { +@TruffleTests class RD9359Test extends Rql2TruffleCompilerTestContext { private val duplicateCsvString = """a,b,a |1,2,3 diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9409Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9409Test.scala index 3c1112992..d1a0e24dd 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9409Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9409Test.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext import raw.client.api._ +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9409Test extends Rql2CompilerTestContext { +@TruffleTests class RD9409Test extends Rql2TruffleCompilerTestContext { test("""let | a = 1, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9445Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9445Test.scala index d55536894..ac7018dc7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9445Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9445Test.scala @@ -14,12 +14,13 @@ package raw.compiler.rql2.tests.regressions import org.scalatest.matchers.{MatchResult, Matcher} import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import java.nio.file.Files import scala.io.Source -trait RD9445Test extends Rql2CompilerTestContext { +@TruffleTests class RD9445Test extends Rql2TruffleCompilerTestContext { def outputAs(expected: String, format: String = "json") = new OutputAs(expected, format) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9479Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9479Test.scala index 4eb641348..8b85ba70f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9479Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9479Test.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.regressions import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9479Test extends Rql2CompilerTestContext { +@TruffleTests class RD9479Test extends Rql2TruffleCompilerTestContext { private val recordData = tempFile("""[{"a": 1, "b": 10, "c": 100}]""") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9485Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9485Test.scala index 3196300ac..711674a44 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9485Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9485Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9485Test extends Rql2CompilerTestContext { +@TruffleTests class RD9485Test extends Rql2TruffleCompilerTestContext { property("raw.inferrer.local.xml.sample-size", "2") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9554Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9554Test.scala index 05136b124..4b72bb27a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9554Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9554Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9554Test extends Rql2CompilerTestContext { +@TruffleTests class RD9554Test extends Rql2TruffleCompilerTestContext { test("""let range = Int.Range(0,5) |in Collection.Union(range, range)""".stripMargin) { it => diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9616Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9616Test.scala index 57a96c891..4fef98532 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9616Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9616Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9616Test extends Rql2CompilerTestContext { +@TruffleTests class RD9616Test extends Rql2TruffleCompilerTestContext { test( """let diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9932Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9932Test.scala index f0feb0b28..21c48ec40 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9932Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/RD9932Test.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.regressions -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD9932Test extends Rql2CompilerTestContext { +@TruffleTests class RD9932Test extends Rql2TruffleCompilerTestContext { val ttt = "\"\"\"" test(s"""Json.Parse( diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD3084Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD3084Test.scala index f4b70bffb..b8eebf7df 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD3084Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD3084Test.scala @@ -12,16 +12,16 @@ package raw.compiler.rql2.tests.regressions.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.jdbc.RDBMSTestCreds +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD3084Test extends Rql2CompilerTestContext with CredentialsTestContext with RDBMSTestCreds { +@TruffleTests class RD3084Test extends Rql2TruffleCompilerTestContext { - rdbms(authorizedUser, "mysql-test", mysqlCreds) - rdbms(authorizedUser, "postgres-test", pgsqlCreds) - rdbms(authorizedUser, "oracle-test", oracleCreds) - rdbms(authorizedUser, "mssql-test", sqlServerCreds) + rdbms("mysql-test", TestCredentials.mysqlCreds) + rdbms("postgres-test", TestCredentials.pgsqlCreds) + rdbms("oracle-test", TestCredentials.oracleCreds) + rdbms("mssql-test", TestCredentials.sqlServerCreds) test("""MySQL.InferAndQuery("mysql-test", "select * from test_types")""") { _ should run diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD4445Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD4445Test.scala index f2965ed45..e718c2cd9 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD4445Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD4445Test.scala @@ -12,42 +12,28 @@ package raw.compiler.rql2.tests.regressions.credentials -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.dropbox.DropboxTestCreds +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD4445Test extends Rql2CompilerTestContext with CredentialsTestContext with DropboxTestCreds { +@TruffleTests class RD4445Test extends Rql2TruffleCompilerTestContext { - dropbox(authorizedUser, dropboxToken) - oauth(authorizedUser, "rawlabs-dropbox", dropboxAccessTokenCredential) + import raw.compiler.rql2.tests.TestCredentials._ - test("""String.ReadLines("dropbox://rawlabs-dropbox/New folder/New Document")""")( - _ should evaluateTo("""["Hello", "World", "Again!"]""") - ) + property("raw.sources.dropbox.clientId", dropboxClientId) - test("""String.ReadLines("dropbox:///New folder/New Document")""")( - _ should evaluateTo("""["Hello", "World", "Again!"]""") - ) + httpHeaders("rawlabs-dropbox", Map("Authorization" -> ("Bearer " + dropboxLongLivedAccessToken))) - test("""String.ReadLines("dropbox:/New folder/New Document")""")( + test("""String.ReadLines("dropbox://rawlabs-dropbox/New folder/New Document")""")( _ should evaluateTo("""["Hello", "World", "Again!"]""") ) - test("""String.ReadLines("dropbox:New folder/New Document")""")( - _ should runErrorAs("""path invalid: New folder/New Document""") - ) - - test("""Location.Ls("dropbox:/New Folder")""")(_ should evaluateTo("""["dropbox:///New Folder/New Document"]""")) - test("""Location.Ls("dropbox:///New Folder")""")(_ should evaluateTo("""["dropbox:///New Folder/New Document"]""")) test("""Location.Ls("dropbox://rawlabs-dropbox/New Folder")""")( - _ should evaluateTo("""["dropbox://rawlabs-dropbox/New Folder/New Document"]""") + _ should evaluateTo("""["dropbox:/New Folder/New Document"]""") ) - //Listing same folder but with trailing '/' - test("""Location.Ls("dropbox:/New Folder/")""")(_ should evaluateTo("""["dropbox:///New Folder/New Document"]""")) - test("""Location.Ls("dropbox:///New Folder/")""")(_ should evaluateTo("""["dropbox:///New Folder/New Document"]""")) + // Listing same folder but with trailing '/' test("""Location.Ls("dropbox://rawlabs-dropbox/New Folder/")""")( - _ should evaluateTo("""["dropbox://rawlabs-dropbox/New Folder/New Document"]""") + _ should evaluateTo("""["dropbox:/New Folder/New Document"]""") ) } diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD5932Test.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD5932Test.scala index a3f92e646..bd1a24738 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD5932Test.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/regressions/credentials/RD5932Test.scala @@ -12,13 +12,13 @@ package raw.compiler.rql2.tests.regressions.credentials -import raw.creds.s3.S3TestCreds -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext +import raw.compiler.rql2.tests.TestCredentials +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait RD5932Test extends Rql2CompilerTestContext with CredentialsTestContext with S3TestCreds { +@TruffleTests class RD5932Test extends Rql2TruffleCompilerTestContext { - s3Bucket(authorizedUser, UnitTestPrivateBucket) + s3Bucket(TestCredentials.UnitTestPrivateBucket, TestCredentials.UnitTestPrivateBucketCred) test("""Json.InferAndRead("s3://rawlabs-private-test-data/rd-5932.json")""") { _ should run diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BasicStagedCompilerTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BasicStagedCompilerTest.scala index c3858bbbe..34a81721b 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BasicStagedCompilerTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BasicStagedCompilerTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BasicStagedCompilerTest extends Rql2CompilerTestContext { +@TruffleTests class BasicStagedCompilerTest extends Rql2TruffleCompilerTestContext { test("""TestPackage.ByteValueArg(22b)""")(_ should evaluateTo("{arg: 22b}")) test("""TestPackage.ShortValueArg(22s)""")(_ should evaluateTo("{arg: 22s}")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpAndTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpAndTest.scala index 674fded00..1a8e34115 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpAndTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpAndTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.spec import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpAndTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class BinaryExpAndTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("""true and true""")(_ should evaluateTo("true")) test("""true and false""")(_ should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpDivTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpDivTest.scala index c1076ed96..76622e9b4 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpDivTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpDivTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpDivTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpDivTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""4 / 2""") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpEqTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpEqTest.scala index 770679b01..e995c99ae 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpEqTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpEqTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpEqTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpEqTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 == 1")(it => it should evaluateTo("true")) test("1 == 2")(it => it should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGeTest.scala index fdb6e4ffa..c360c51c7 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGeTest.scala @@ -12,10 +12,11 @@ package raw.compiler.rql2.tests.spec +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext -trait BinaryExpGeTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpGeTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 >= 1")(it => it should evaluateTo("true")) test("1 >= 2")(it => it should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGtTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGtTest.scala index 2dd2dbf67..24e348673 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGtTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpGtTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpGtTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpGtTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 > 1")(it => it should evaluateTo("false")) test("1 > 2")(it => it should evaluateTo("false")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLeTest.scala index e1183e8f5..176fd3e39 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLeTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpLeTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpLeTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 <= 1")(it => it should evaluateTo("true")) test("1 <= 2")(it => it should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLtTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLtTest.scala index fb7250644..ce708d5a9 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLtTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpLtTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpLtTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpLtTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 < 1")(it => it should evaluateTo("false")) test("1 < 2")(it => it should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpModTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpModTest.scala index 6012509bf..fa8b33327 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpModTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpModTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpModTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpModTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""5 % 3""") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpMultTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpMultTest.scala index 06bf4d565..ac72e189f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpMultTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpMultTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpMultTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpMultTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""1 * 1""") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpNeqTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpNeqTest.scala index a7924c6af..38856f8d3 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpNeqTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpNeqTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpNeqTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpNeqTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("1 != 1")(it => it should evaluateTo("false")) test("1 != 2")(it => it should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpOrTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpOrTest.scala index 1b7019d29..1eca03e88 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpOrTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpOrTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.spec import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpOrTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class BinaryExpOrTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("""true or true""")(_ should evaluateTo("true")) test("""true or false""")(_ should evaluateTo("true")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpPlusTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpPlusTest.scala index fa7caba19..5ea87266b 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpPlusTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpPlusTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpPlusTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpPlusTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""1 + 1""") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpSubTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpSubTest.scala index cc652cdec..0eeb6db46 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpSubTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/BinaryExpSubTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait BinaryExpSubTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class BinaryExpSubTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""1 - 1""") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ClosureTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ClosureTest.scala index f9a5df2eb..010014f16 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ClosureTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ClosureTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ClosureTest extends Rql2CompilerTestContext { +@TruffleTests class ClosureTest extends Rql2TruffleCompilerTestContext { test("""let functions = | List.Transform([1,2,3], diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ConstTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ConstTest.scala index bfda3c890..827ba07b2 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ConstTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ConstTest.scala @@ -15,9 +15,10 @@ package raw.compiler.rql2.tests.spec import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData import raw.compiler.rql2.source.{Rql2BoolType, Rql2IntType, Rql2StringType} -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ConstTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class ConstTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("1") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ErrorsTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ErrorsTest.scala index 37e098eb0..de037490f 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ErrorsTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ErrorsTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ErrorsTest extends Rql2CompilerTestContext { +@TruffleTests class ErrorsTest extends Rql2TruffleCompilerTestContext { test("""Collection.Count("a")""")(_ shouldNot tipe) test("""Collection.Count(Collection.Count("a"))""")(_ shouldNot tipe) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/FunAbsTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/FunAbsTest.scala index 68d8ef349..30152a0c3 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/FunAbsTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/FunAbsTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait FunAbsTest extends Rql2CompilerTestContext { +@TruffleTests class FunAbsTest extends Rql2TruffleCompilerTestContext { test(""" |let f = (v: int) -> v + 1 diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/IfThenElseTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/IfThenElseTest.scala index c3b49c09e..8d1955312 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/IfThenElseTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/IfThenElseTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait IfThenElseTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class IfThenElseTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test(""" |if true then "a" else "b" diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ImplicitCastTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ImplicitCastTest.scala index 42274d60f..c0bfeddd8 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ImplicitCastTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ImplicitCastTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ImplicitCastTest extends Rql2CompilerTestContext { +@TruffleTests class ImplicitCastTest extends Rql2TruffleCompilerTestContext { // compatible options as function param test(s"""let f = (x: int) -> Record.Build(o = x) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/JoinWithTryRowsTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/JoinWithTryRowsTest.scala index b716b226a..f06302046 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/JoinWithTryRowsTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/JoinWithTryRowsTest.scala @@ -12,11 +12,12 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext import raw.sources.filesystem.local.LocalLocationsTestContext import raw.compiler.utils._ +import raw.testing.tags.TruffleTests -trait JoinWithTryRowsTest extends Rql2CompilerTestContext with LocalLocationsTestContext { +@TruffleTests class JoinWithTryRowsTest extends Rql2TruffleCompilerTestContext with LocalLocationsTestContext { /* testing joins with failed rows. How a dataset of failed rows is built: by engineering a dataset of nested list of records, and pick one using List.Get so that it sometimes diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetBindTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetBindTest.scala index 3e72b970c..911ec5525 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetBindTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetBindTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LetBindTest extends Rql2CompilerTestContext with CombinationSpecTestHelper { +@TruffleTests class LetBindTest extends Rql2TruffleCompilerTestContext with CombinationSpecTestHelper { test("""let x = 1 |in x diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunRecTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunRecTest.scala index 6f5209a11..cdc670b82 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunRecTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunRecTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.compiler.rql2.errors.OutputTypeRequiredForRecursiveFunction -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LetFunRecTest extends Rql2CompilerTestContext { +@TruffleTests class LetFunRecTest extends Rql2TruffleCompilerTestContext { test("""let rec fact(v: int): int = if (v > 1) then v * fact(v - 1) else v |in fact(4) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunTest.scala index 1f19e4128..33f2cbc58 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetFunTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LetFunTest extends Rql2CompilerTestContext { +@TruffleTests class LetFunTest extends Rql2TruffleCompilerTestContext { test("""let f(x: int) = x + 1 |in f(0) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetTypeTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetTypeTest.scala index 171624e1b..8f49a4cb6 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetTypeTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/LetTypeTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait LetTypeTest extends Rql2CompilerTestContext { +@TruffleTests class LetTypeTest extends Rql2TruffleCompilerTestContext { test(""" |let t = type int, diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/MigrationTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/MigrationTest.scala index 1cedbba0e..7ff920471 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/MigrationTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/MigrationTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.compiler.rql2.source.{Rql2IntType, Rql2IsNullableTypeProperty, Rql2IsTryableTypeProperty, Rql2UndefinedType} -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait MigrationTest extends Rql2CompilerTestContext { +@TruffleTests class MigrationTest extends Rql2TruffleCompilerTestContext { test("1") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PackageNameTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PackageNameTest.scala index f615dc224..b3271cc39 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PackageNameTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PackageNameTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait PackageNameTest extends Rql2CompilerTestContext { +@TruffleTests class PackageNameTest extends Rql2TruffleCompilerTestContext { test("""Record""")(it => it should typeAs("""package("Record")""")) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ProjTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ProjTest.scala index 7f8851ddc..3cd211759 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ProjTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/ProjTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait ProjTest extends Rql2CompilerTestContext { +@TruffleTests class ProjTest extends Rql2TruffleCompilerTestContext { test("""let a = {x: 1} |in a.x diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PropagationTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PropagationTest.scala index 70b26288c..7289e3786 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PropagationTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/PropagationTest.scala @@ -12,9 +12,10 @@ package raw.compiler.rql2.tests.spec -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait PropagationTest extends Rql2CompilerTestContext { +@TruffleTests class PropagationTest extends Rql2TruffleCompilerTestContext { // lists test("""let l = List.Build(1,2,3,2,1) |in TestPackage.StrictArgs(l)""".stripMargin)(_ should (typeAs("float") and evaluateTo("5.0f"))) diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/StagedCompilerTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/StagedCompilerTest.scala index c05ce75ac..c03325c9d 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/StagedCompilerTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/StagedCompilerTest.scala @@ -13,9 +13,10 @@ package raw.compiler.rql2.tests.spec import raw.compiler.utils._ -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait StagedCompilerTest extends Rql2CompilerTestContext { +@TruffleTests class StagedCompilerTest extends Rql2TruffleCompilerTestContext { val data = tempFile(""" |[ diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNegTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNegTest.scala index 5077d9abf..f9d6f630a 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNegTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNegTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.spec import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait UnaryExpNegTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class UnaryExpNegTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("- 1") { it => it should typeAs("int") diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNotTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNotTest.scala index 882dabeeb..93f8fe7ce 100644 --- a/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNotTest.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/tests/spec/UnaryExpNotTest.scala @@ -14,9 +14,10 @@ package raw.compiler.rql2.tests.spec import org.scalatest.prop.TableDrivenPropertyChecks import raw.utils.TestData -import raw.compiler.rql2.tests.Rql2CompilerTestContext +import raw.compiler.rql2.truffle.Rql2TruffleCompilerTestContext +import raw.testing.tags.TruffleTests -trait UnaryExpNotTest extends Rql2CompilerTestContext with TableDrivenPropertyChecks { +@TruffleTests class UnaryExpNotTest extends Rql2TruffleCompilerTestContext with TableDrivenPropertyChecks { test("""not true""".stripMargin) { it => it should typeAs("bool") it should evaluateTo("false") diff --git a/sources/src/main/scala/raw/creds/api/CredentialsException.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerTestContext.scala similarity index 65% rename from sources/src/main/scala/raw/creds/api/CredentialsException.scala rename to snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerTestContext.scala index 14a336852..8a70139c5 100644 --- a/sources/src/main/scala/raw/creds/api/CredentialsException.scala +++ b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/Rql2TruffleCompilerTestContext.scala @@ -10,9 +10,8 @@ * licenses/APL.txt. */ -package raw.creds.api +package raw.compiler.rql2.truffle -import raw.utils.RawServiceException +import raw.compiler.rql2.tests.Rql2CompilerTestContext -class CredentialsException(message: String, cause: Throwable = null) - extends RawServiceException(s"credentials error: $message", cause) +trait Rql2TruffleCompilerTestContext extends Rql2CompilerTestContext with Rql2TruffleCompilerServiceTestContext diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleWithLocalCredentialsTestContext.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleWithLocalCredentialsTestContext.scala deleted file mode 100644 index c434ee979..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/TruffleWithLocalCredentialsTestContext.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle - -import raw.compiler.rql2.tests.Rql2CompilerTestContext -import raw.creds.api.CredentialsTestContext -import raw.creds.local.LocalCredentialsTestContext - -trait TruffleWithLocalCredentialsTestContext - extends Rql2CompilerTestContext - with LocalCredentialsTestContext - with CredentialsTestContext - with Rql2TruffleCompilerServiceTestContext diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/benchmark/TruffleBenchmarkTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/benchmark/TruffleBenchmarkTest.scala deleted file mode 100644 index 78dc43859..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/benchmark/TruffleBenchmarkTest.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.benchmark - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.benchmark._ - -@TruffleTests class BenchmarkTruffleTests extends TruffleWithLocalCredentialsTestContext with BenchmarkTests - -@TruffleTests class StressTruffleTests extends TruffleWithLocalCredentialsTestContext with StressTests diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/TruffleBuiltInTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/TruffleBuiltInTest.scala deleted file mode 100644 index 7f72a2b79..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/TruffleBuiltInTest.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.builtin - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.builtin._ - -@TruffleTests class BinaryPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryPackageTest -@TruffleTests class BytePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with BytePackageTest -@TruffleTests class CsvPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with CsvPackageTest -@TruffleTests class DatePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with DatePackageTest -@TruffleTests class DecimalPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with DecimalPackageTest -@TruffleTests class DoublePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with DoublePackageTest -@TruffleTests class EnvironmentPackageTruffleTest - extends TruffleWithLocalCredentialsTestContext - with EnvironmentPackageTest -@TruffleTests class ErrorPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with ErrorPackageTest -@TruffleTests class FloatPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with FloatPackageTest -@TruffleTests class FunctionPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with FunctionPackageTest -@TruffleTests class HttpPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with HttpPackageTest -@TruffleTests class IntervalPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with IntervalPackageTest -@TruffleTests class IntPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with IntPackageTest -@TruffleTests class JsonPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with JsonPackageTest -@TruffleTests class LongPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with LongPackageTest -@TruffleTests class MathPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with MathPackageTest -@TruffleTests class NullablePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with NullablePackageTest -@TruffleTests class NullableTryablePackageTruffleTest - extends TruffleWithLocalCredentialsTestContext - with NullableTryablePackageTest -@TruffleTests class RecordPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with RecordPackageTest -@TruffleTests class RegexPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with RegexPackageTest -@TruffleTests class S3PackageTruffleTest extends TruffleWithLocalCredentialsTestContext with S3PackageTest -@TruffleTests class ShortPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with ShortPackageTest -@TruffleTests class StringPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with StringPackageTest -@TruffleTests class SuccessPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with SuccessPackageTest -@TruffleTests class TimePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with TimePackageTest -@TruffleTests class TimestampPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with TimestampPackageTest -@TruffleTests class TryPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with TryPackageTest -@TruffleTests class TypePackageTruffleTest extends TruffleWithLocalCredentialsTestContext with TypePackageTest -@TruffleTests class XmlPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with XmlPackageTest -@TruffleTests class LocationPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with LocationPackageTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/collection/TruffleCollectionTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/collection/TruffleCollectionTest.scala deleted file mode 100644 index a79959abe..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/collection/TruffleCollectionTest.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.builtin.collection - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.builtin.collection._ - -@TruffleTests class CollectionDistinctTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionDistinctTest - -@TruffleTests class CollectionExplodeTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionExplodeTest - -@TruffleTests class CollectionGroupByTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionGroupByTest - -@TruffleTests class CollectionJoinTruffleTest extends TruffleWithLocalCredentialsTestContext with CollectionJoinTest -@TruffleTests class CollectionMinMaxTruffleTest extends TruffleWithLocalCredentialsTestContext with CollectionMinMaxTest -@TruffleTests class CollectionMkStringTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionMkStringTest - -@TruffleTests class CollectionOrderByTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionOrderByTest - -@TruffleTests class CollectionPackageTruffleTest - extends TruffleWithLocalCredentialsTestContext - with CollectionPackageTest - -@TruffleTests class CollectionRangeTruffleTest extends TruffleWithLocalCredentialsTestContext with CollectionRangeTest -@TruffleTests class CollectionUnionTruffleTest extends TruffleWithLocalCredentialsTestContext with CollectionUnionTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/list/TruffleListTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/list/TruffleListTest.scala deleted file mode 100644 index fb3faee80..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/builtin/list/TruffleListTest.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.builtin.list - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.builtin.list._ - -@TruffleTests class ListDistinctTruffleTest extends TruffleWithLocalCredentialsTestContext with ListDistinctTest - -@TruffleTests class ListExplodeTruffleTest extends TruffleWithLocalCredentialsTestContext with ListExplodeTest - -@TruffleTests class ListGroupByTruffleTest extends TruffleWithLocalCredentialsTestContext with ListGroupByTest -@TruffleTests class ListJoinTruffleTest extends TruffleWithLocalCredentialsTestContext with ListJoinTest -@TruffleTests class ListMinMaxTruffleTest extends TruffleWithLocalCredentialsTestContext with ListMinMaxTest -@TruffleTests class ListMkStringTruffleTest extends TruffleWithLocalCredentialsTestContext with ListMkStringTest -@TruffleTests class ListOrderByTruffleTest extends TruffleWithLocalCredentialsTestContext with ListOrderByTest -@TruffleTests class ListPackageTruffleTest extends TruffleWithLocalCredentialsTestContext with ListPackageTest -@TruffleTests class ListUnionTruffleTest extends TruffleWithLocalCredentialsTestContext with ListUnionTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/hints/TruffleHintsTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/hints/TruffleHintsTest.scala deleted file mode 100644 index b316cfea1..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/hints/TruffleHintsTest.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.hints - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.hints._ - -@TruffleTests class SemanticAnalyzerHintsTruffleTest - extends TruffleWithLocalCredentialsTestContext - with SemanticAnalyzerHintsTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/lsp/TruffleLspTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/lsp/TruffleLspTest.scala deleted file mode 100644 index b6985069d..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/lsp/TruffleLspTest.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.lsp - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.lsp._ - -@TruffleTests class LspBrokenCodeTruffleTest extends TruffleWithLocalCredentialsTestContext with LspBrokenCodeTest -@TruffleTests class LspCommentsFormatTruffleTest - extends TruffleWithLocalCredentialsTestContext - with LspCommentsFormatTest -@TruffleTests class LspDefinitionTruffleTest extends TruffleWithLocalCredentialsTestContext with LspDefinitionTest -@TruffleTests class LspDotAutoCompleteTruffleTest - extends TruffleWithLocalCredentialsTestContext - with LspDotAutoCompleteTest -@TruffleTests class LspFormatCodeTruffleTest extends TruffleWithLocalCredentialsTestContext with LspFormatCodeTest -@TruffleTests class LspHoverTruffleTest extends TruffleWithLocalCredentialsTestContext with LspHoverTest -@TruffleTests class LspRenameTruffleTest extends TruffleWithLocalCredentialsTestContext with LspRenameTest -@TruffleTests class LspValidateTruffleTest extends TruffleWithLocalCredentialsTestContext with LspValidateTest -@TruffleTests class LspWordAutoCompleteTruffleTest - extends TruffleWithLocalCredentialsTestContext - with LspWordAutoCompleteTest -@TruffleTests class LspCompilationMessagesTruffleTest - extends TruffleWithLocalCredentialsTestContext - with LspCompilationMessagesTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/TruffleOffHeapTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/TruffleOffHeapTest.scala deleted file mode 100644 index 79cae32cc..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/offheap/TruffleOffHeapTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.offheap - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.offheap._ - -@TruffleTests class OffHeapDistinctTruffleTest extends TruffleWithLocalCredentialsTestContext with OffHeapDistinctTest -@TruffleTests class OffHeapEquiJoinTruffleTest extends TruffleWithLocalCredentialsTestContext with OffHeapEquiJoinTest -@TruffleTests class OffHeapGroupTruffleTest extends TruffleWithLocalCredentialsTestContext with OffHeapGroupTest -@TruffleTests class OffHeapJoinTruffleTest extends TruffleWithLocalCredentialsTestContext with OffHeapJoinTest -@TruffleTests class OffHeapOrderByTruffleTest extends TruffleWithLocalCredentialsTestContext with OffHeapOrderByTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/output/TruffleOutputTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/output/TruffleOutputTest.scala deleted file mode 100644 index f76a2a33c..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/output/TruffleOutputTest.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.output - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.output._ - -@TruffleTests class BinaryOutputTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryOutputTest -@TruffleTests class TextOutputTruffleTest extends TruffleWithLocalCredentialsTestContext with TextOutputTest -@TruffleTests class CsvOutputTruffleTest extends TruffleWithLocalCredentialsTestContext with CsvOutputTest -@TruffleTests class JsonOutputTruffleTest extends TruffleWithLocalCredentialsTestContext with JsonOutputTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/parser/TruffleParserTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/parser/TruffleParserTest.scala deleted file mode 100644 index e7563f6ae..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/parser/TruffleParserTest.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.parser - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.parser._ - -@TruffleTests class FrontendSyntaxAnalyzerTruffleTest - extends TruffleWithLocalCredentialsTestContext - with FrontendSyntaxAnalyzerTest -@TruffleTests class ListSugarTruffleTest extends TruffleWithLocalCredentialsTestContext with ListSugarTest -@TruffleTests class OperatorPrecedenceTruffleTest - extends TruffleWithLocalCredentialsTestContext - with OperatorPrecedenceTest -@TruffleTests class RecordSugarTruffleTest extends TruffleWithLocalCredentialsTestContext with RecordSugarTest diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/TruffleRegressionTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/TruffleRegressionTest.scala deleted file mode 100644 index 111d04e15..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/regressions/TruffleRegressionTest.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.regressions - -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests -import raw.compiler.rql2.tests.regressions._ - -@TruffleTests class RD9932TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9932Test -@TruffleTests class RD9485TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9485Test -@TruffleTests class RD9445TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9445Test -@TruffleTests class RD5797TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5797Test -@TruffleTests class RD5779TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5779Test -@TruffleTests class RD5393TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5393Test -@TruffleTests class RD5884TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5884Test -@TruffleTests class RD5448TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5448Test -@TruffleTests class RD5238TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5238Test -@TruffleTests class RD3742TruffleTest extends TruffleWithLocalCredentialsTestContext with RD3742Test -@TruffleTests class RD3784TruffleTest extends TruffleWithLocalCredentialsTestContext with RD3784Test -@TruffleTests class RD5784TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5784Test -@TruffleTests class RD4981TruffleTest extends TruffleWithLocalCredentialsTestContext with RD4981Test -@TruffleTests class RD5920TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5920Test -@TruffleTests class RD572TruffleTest extends TruffleWithLocalCredentialsTestContext with RD572Test -@TruffleTests class RD5785TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5785Test -@TruffleTests class RD5491TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5491Test -@TruffleTests class RD7974TruffleTest extends TruffleWithLocalCredentialsTestContext with RD7974Test -@TruffleTests class RD5714TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5714Test -@TruffleTests class RD5697TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5697Test -@TruffleTests class RD5644TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5644Test -@TruffleTests class RD8993TruffleTest extends TruffleWithLocalCredentialsTestContext with RD8993Test -@TruffleTests class RD4529TruffleTest extends TruffleWithLocalCredentialsTestContext with RD4529Test -@TruffleTests class RD5893TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5893Test -@TruffleTests class RD8764TruffleTest extends TruffleWithLocalCredentialsTestContext with RD8764Test -@TruffleTests class RD5979TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5979Test -@TruffleTests class RD5679TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5679Test -@TruffleTests class RD5775TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5775Test -@TruffleTests class RD5851TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5851Test -@TruffleTests class RD5412TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5412Test -@TruffleTests class RD5971TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5971Test -@TruffleTests class RD5968TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5968Test -@TruffleTests class RD5691TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5691Test -@TruffleTests class RD8935TruffleTest extends TruffleWithLocalCredentialsTestContext with RD8935Test -@TruffleTests class RD7924TruffleTest extends TruffleWithLocalCredentialsTestContext with RD7924Test -@TruffleTests class RD5488TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5488Test -@TruffleTests class RD5786TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5786Test -@TruffleTests class RD5722TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5722Test -@TruffleTests class RD5925TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5925Test -@TruffleTests class RD5921TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5921Test -@TruffleTests class RD5932TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5932Test -@TruffleTests class RD5484TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5484Test -@TruffleTests class RD8530TruffleTest extends TruffleWithLocalCredentialsTestContext with RD8530Test -@TruffleTests class RD5685TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5685Test -@TruffleTests class RD5365TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5365Test -@TruffleTests class RD5914TruffleTest extends TruffleWithLocalCredentialsTestContext with RD5914Test -@TruffleTests class RD9137TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9137Test -@TruffleTests class RD9228TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9228Test -@TruffleTests class RD9359TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9359Test -@TruffleTests class RD9255TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9255Test -@TruffleTests class RD9229TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9229Test -@TruffleTests class RD9409TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9409Test -@TruffleTests class RD9479TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9479Test -@TruffleTests class RD9554TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9554Test -@TruffleTests class RD9616TruffleTest extends TruffleWithLocalCredentialsTestContext with RD9616Test -@TruffleTests class RD10194TruffleTest extends TruffleWithLocalCredentialsTestContext with RD10194Test -@TruffleTests class RD10220TruffleTest extends TruffleWithLocalCredentialsTestContext with RD10220Test -@TruffleTests class RD10723TruffleTest extends TruffleWithLocalCredentialsTestContext with RD10723Test -@TruffleTests class RD10801TruffleTest extends TruffleWithLocalCredentialsTestContext with RD10801Test diff --git a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/spec/TruffleSpecTest.scala b/snapi-client/src/test/scala/raw/compiler/rql2/truffle/spec/TruffleSpecTest.scala deleted file mode 100644 index 0aee50ef4..000000000 --- a/snapi-client/src/test/scala/raw/compiler/rql2/truffle/spec/TruffleSpecTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.truffle.spec - -import raw.compiler.rql2.tests.spec._ -import raw.compiler.rql2.truffle.TruffleWithLocalCredentialsTestContext -import raw.testing.tags.TruffleTests - -@TruffleTests class PropagationTruffleTest extends TruffleWithLocalCredentialsTestContext with PropagationTest -@TruffleTests class JoinWithTryRowsTruffleTest extends TruffleWithLocalCredentialsTestContext with JoinWithTryRowsTest -@TruffleTests class ConstTruffleTest extends TruffleWithLocalCredentialsTestContext with ConstTest -@TruffleTests class BinaryExpLeTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpLeTest -@TruffleTests class BinaryExpAndTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpAndTest -@TruffleTests class UnaryExpNotTruffleTest extends TruffleWithLocalCredentialsTestContext with UnaryExpNotTest -@TruffleTests class ProjTruffleTest extends TruffleWithLocalCredentialsTestContext with ProjTest -@TruffleTests class ErrorsTruffleTest extends TruffleWithLocalCredentialsTestContext with ErrorsTest - -@TruffleTests class IfThenElseTruffleTest extends TruffleWithLocalCredentialsTestContext with IfThenElseTest -@TruffleTests class BinaryExpGtTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpGtTest -@TruffleTests class BinaryExpMultTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpMultTest -@TruffleTests class BinaryExpGeTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpGeTest -@TruffleTests class UnaryExpNegTruffleTest extends TruffleWithLocalCredentialsTestContext with UnaryExpNegTest -@TruffleTests class BinaryExpDivTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpDivTest -@TruffleTests class MigrationTruffleTest extends TruffleWithLocalCredentialsTestContext with MigrationTest - -@TruffleTests class LetBindTruffleTest extends TruffleWithLocalCredentialsTestContext with LetBindTest -@TruffleTests class FunAbsTruffleTest extends TruffleWithLocalCredentialsTestContext with FunAbsTest - -@TruffleTests class LetFunRecTruffleTest extends TruffleWithLocalCredentialsTestContext with LetFunRecTest -@TruffleTests class ClosureTruffleTest extends TruffleWithLocalCredentialsTestContext with ClosureTest -@TruffleTests class BinaryExpOrTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpOrTest -@TruffleTests class BinaryExpEqTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpEqTest -@TruffleTests class BinaryExpNeqTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpNeqTest -@TruffleTests class ImplicitCastTruffleTest extends TruffleWithLocalCredentialsTestContext with ImplicitCastTest -@TruffleTests class BinaryExpModTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpModTest -@TruffleTests class BinaryExpSubTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpSubTest -@TruffleTests class LetTypeTruffleTest extends TruffleWithLocalCredentialsTestContext with LetTypeTest -@TruffleTests class BinaryExpPlusTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpPlusTest -@TruffleTests class BasicStagedCompilerTruffleTest - extends TruffleWithLocalCredentialsTestContext - with BasicStagedCompilerTest -@TruffleTests class BinaryExpLtTruffleTest extends TruffleWithLocalCredentialsTestContext with BinaryExpLtTest -@TruffleTests class PackageNameTruffleTest extends TruffleWithLocalCredentialsTestContext with PackageNameTest - -@TruffleTests class StagedCompilerTruffleTest extends TruffleWithLocalCredentialsTestContext with StagedCompilerTest - -@TruffleTests class LetFunTruffleTest extends TruffleWithLocalCredentialsTestContext with LetFunTest diff --git a/snapi-frontend/src/main/java/module-info.java b/snapi-frontend/src/main/java/module-info.java index 8f44366d5..71adeb22e 100644 --- a/snapi-frontend/src/main/java/module-info.java +++ b/snapi-frontend/src/main/java/module-info.java @@ -17,6 +17,8 @@ requires com.fasterxml.jackson.databind; requires com.fasterxml.jackson.dataformat.csv; requires com.fasterxml.jackson.scala; + requires com.fasterxml.jackson.datatype.jsr310; + requires com.fasterxml.jackson.datatype.jdk8; requires java.xml; requires java.sql; requires com.ctc.wstx; @@ -34,8 +36,6 @@ requires ch.qos.logback.classic; requires com.google.common; requires jul.to.slf4j; - requires com.fasterxml.jackson.datatype.jsr310; - requires com.fasterxml.jackson.datatype.jdk8; requires org.graalvm.polyglot; requires raw.utils; requires raw.client; @@ -60,4 +60,6 @@ opens raw.inferrer.api to com.fasterxml.jackson.databind; + opens raw.compiler.rql2.api to + com.fasterxml.jackson.databind; } diff --git a/snapi-frontend/src/main/scala/raw/compiler/base/CompilerContext.scala b/snapi-frontend/src/main/scala/raw/compiler/base/CompilerContext.scala index a9ccfa002..033278038 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/base/CompilerContext.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/base/CompilerContext.scala @@ -13,7 +13,6 @@ package raw.compiler.base import com.typesafe.scalalogging.StrictLogging -import raw.sources.api.SourceContext import raw.inferrer.api.{InferrerProperties, InferrerService, InputFormatDescriptor} import raw.utils._ @@ -22,9 +21,8 @@ import raw.utils._ */ class CompilerContext( val language: String, - val user: AuthenticatedUser, - val inferrer: InferrerService, - val sourceContext: SourceContext + val user: RawUid, + val inferrer: InferrerService )( implicit val settings: RawSettings ) extends StrictLogging { diff --git a/snapi-frontend/src/main/scala/raw/compiler/base/NormalizeMap.scala b/snapi-frontend/src/main/scala/raw/compiler/base/NormalizeMap.scala deleted file mode 100644 index eab9dff49..000000000 --- a/snapi-frontend/src/main/scala/raw/compiler/base/NormalizeMap.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.base - -import org.bitbucket.inkytonik.kiama.util.Entity -import raw.compiler.base.source.BaseIdnNode - -import scala.collection.mutable - -class NormalizeMap(prefix: String = "") { - - private var cntIdnNode = 0 - private val mapIdnNode = mutable.HashMap[Entity, String]() - - def newIdnForIdnNode(entity: Entity, n: BaseIdnNode): String = { - - mapIdnNode.getOrElseUpdate( - entity, { - cntIdnNode += 1 - s"${prefix}_$$$cntIdnNode" - } - ) - } - - private var cntString = 0 - private val mapString = mutable.HashMap[String, String]() - - def newIdnForString(n: String): String = { - mapString.getOrElseUpdate( - n, { - cntString += 1 - s"${prefix}_$$$cntString" - } - ) - } - -} diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/ImplicitCasts.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/ImplicitCasts.scala index 2f685e6d6..6bf00e8bd 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/ImplicitCasts.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/ImplicitCasts.scala @@ -144,7 +144,7 @@ class ImplicitCasts(protected val parent: Phase[SourceProgram], protected val ph case _ => None } case _: Rql2LocationType => actual match { - case _: Rql2StringType => Some(LocationPackageBuilder.Build(e, Vector.empty)) + case _: Rql2StringType => Some(LocationPackageBuilder.FromString(e)) case _ => None } case Rql2IterableType(t, _) => actual match { @@ -418,7 +418,6 @@ class ImplicitCasts(protected val parent: Phase[SourceProgram], protected val ph } } case _ => fail - // TODO (msb): LegacyCallLanguage args })) val r = rewrite(s)(tree.root) diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/Propagation.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/Propagation.scala index e5dcdc5bf..f6d4b69ee 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/Propagation.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/Propagation.scala @@ -17,26 +17,7 @@ import org.bitbucket.inkytonik.kiama.rewriting.Strategy import raw.compiler.base.Phase import raw.compiler.base.source.Type import raw.compiler.common.source._ -import raw.compiler.rql2.api.{ - Arg, - Rql2BoolValue, - Rql2ByteValue, - Rql2DateValue, - Rql2DoubleValue, - Rql2FloatValue, - Rql2IntValue, - Rql2IntervalValue, - Rql2ListValue, - Rql2LongValue, - Rql2OptionValue, - Rql2RecordValue, - Rql2ShortValue, - Rql2StringValue, - Rql2TimeValue, - Rql2TimestampValue, - Rql2Value, - ValueArg -} +import raw.compiler.rql2.api.{Arg, Rql2Value, ValueArg} import raw.compiler.rql2.builtin._ import raw.compiler.rql2.source._ @@ -346,43 +327,6 @@ class Propagation(protected val parent: Phase[SourceProgram], protected val phas r } - private def valueToExp(value: Rql2Value, t: Type): Exp = value match { - case Rql2ByteValue(v) => ByteConst(v.toString) - case Rql2ShortValue(v) => ShortConst(v.toString) - case Rql2IntValue(v) => IntConst(v.toString) - case Rql2LongValue(v) => LongConst(v.toString) - case Rql2FloatValue(v) => FloatConst(v.toString) - case Rql2DoubleValue(v) => DoubleConst(v.toString) - case Rql2StringValue(v) => StringConst(v) - case Rql2BoolValue(v) => BoolConst(v) - case Rql2OptionValue(option) => - val innerType = resetProps(t, Set.empty) - option - .map(v => valueToExp(v, innerType)) - .map(NullablePackageBuilder.Build(_)) - .getOrElse(NullablePackageBuilder.Empty(innerType)) - case Rql2RecordValue(r) => - val Rql2RecordType(atts, _) = t - val fields = r.zip(atts).map { case (v, att) => att.idn -> valueToExp(v, att.tipe) } - RecordPackageBuilder.Build(fields.toVector) - case Rql2ListValue(v) => - val Rql2ListType(innerType, _) = t - ListPackageBuilder.Build(v.map(x => valueToExp(x, innerType)): _*) - case Rql2DateValue(v) => DatePackageBuilder.FromLocalDate(v) - case Rql2TimeValue(v) => TimePackageBuilder.FromLocalTime(v) - case Rql2TimestampValue(v) => TimestampPackageBuilder.FromLocalDateTime(v) - case Rql2IntervalValue( - years, - month, - weeks, - days, - hours, - minutes, - seconds, - millis - ) => IntervalPackageBuilder.FromRawInterval(years, month, weeks, days, hours, minutes, seconds, millis) - } - private def sameModuloAttributes(t1: Type, t2: Type) = { val plainType1 = resetProps(t1, Set.empty) val plainType2 = resetProps(t2, Set.empty) diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/Rql2TypeUtils.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/Rql2TypeUtils.scala index 50b16148b..968b94018 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/Rql2TypeUtils.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/Rql2TypeUtils.scala @@ -16,6 +16,34 @@ import org.bitbucket.inkytonik.kiama.rewriting.Rewriter.{everywhere, query} import raw.client.api._ import raw.compiler.base.source.{AnythingType, Type} import raw.compiler.common.source._ +import raw.compiler.rql2.api.{ + Rql2BoolValue, + Rql2ByteValue, + Rql2DateValue, + Rql2DoubleValue, + Rql2FloatValue, + Rql2IntValue, + Rql2IntervalValue, + Rql2ListValue, + Rql2LongValue, + Rql2OptionValue, + Rql2RecordAttr, + Rql2RecordValue, + Rql2ShortValue, + Rql2StringValue, + Rql2TimeValue, + Rql2TimestampValue, + Rql2Value +} +import raw.compiler.rql2.builtin.{ + DatePackageBuilder, + IntervalPackageBuilder, + ListPackageBuilder, + NullablePackageBuilder, + RecordPackageBuilder, + TimePackageBuilder, + TimestampPackageBuilder +} import raw.compiler.rql2.source._ import raw.inferrer.api._ @@ -165,6 +193,44 @@ trait Rql2TypeUtils { } } + final def valueToExp(value: Rql2Value, t: Type): Exp = value match { + case Rql2ByteValue(v) => ByteConst(v.toString) + case Rql2ShortValue(v) => ShortConst(v.toString) + case Rql2IntValue(v) => IntConst(v.toString) + case Rql2LongValue(v) => LongConst(v.toString) + case Rql2FloatValue(v) => FloatConst(v.toString) + case Rql2DoubleValue(v) => DoubleConst(v.toString) + case Rql2StringValue(v) => StringConst(v) + case Rql2BoolValue(v) => BoolConst(v) + case Rql2OptionValue(option) => + val innerType = resetProps(t, Set.empty) + option + .map(v => valueToExp(v, innerType)) + .map(NullablePackageBuilder.Build(_)) + .getOrElse(NullablePackageBuilder.Empty(innerType)) + case Rql2RecordValue(vs) => + val Rql2RecordType(atts, _) = t + RecordPackageBuilder.Build( + vs.zip(atts).map { case (Rql2RecordAttr(idn, v1), att) => idn -> valueToExp(v1, att.tipe) }.toVector + ) + case Rql2ListValue(v) => + val Rql2ListType(innerType, _) = t + ListPackageBuilder.Build(v.map(x => valueToExp(x, innerType)): _*) + case Rql2DateValue(v) => DatePackageBuilder.FromLocalDate(v) + case Rql2TimeValue(v) => TimePackageBuilder.FromLocalTime(v) + case Rql2TimestampValue(v) => TimestampPackageBuilder.FromLocalDateTime(v) + case Rql2IntervalValue( + years, + month, + weeks, + days, + hours, + minutes, + seconds, + millis + ) => IntervalPackageBuilder.FromRawInterval(years, month, weeks, days, hours, minutes, seconds, millis) + } + } object Rql2TypeUtils extends Rql2TypeUtils diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/SemanticAnalyzer.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/SemanticAnalyzer.scala index 4fd59e74b..eb52493c5 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/SemanticAnalyzer.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/SemanticAnalyzer.scala @@ -544,8 +544,9 @@ class SemanticAnalyzer(val tree: SourceTree.SourceTree)(implicit programContext: // Use getValue to confirm Environment.Secret succeeds. The getValue function also handles potentially unexpected nullables and tryables. // We give it a dummy report which states the expected type is the actual type, so that is skips these checks val report = CompatibilityReport(tipe(e), tipe(e)) - // Try execute "Environment.Secret()" - getValue(report, e) match { + // Try execute "Environment.Secret()" + val v = getValue(report, e) + v match { // If getValue returns an error which means the staged compiler failed to execute "Environment.Secret()" code // We return a warning that the secret is missing. case Right(Rql2TryValue(Left(error))) => Seq(MissingSecretWarning(e)) @@ -838,6 +839,7 @@ class SemanticAnalyzer(val tree: SourceTree.SourceTree)(implicit programContext: case _: TripleQuotedStringConst => Rql2StringType() case _: BoolConst => Rql2BoolType() case _: BinaryConst => Rql2BinaryType() + case _: LocationConst => Rql2LocationType() } case l: Let => actualType(l.e) case TypeExp(t) => ExpType(t) diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/StagedCompiler.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/StagedCompiler.scala index be2be3151..abc0f97a4 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/StagedCompiler.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/StagedCompiler.scala @@ -13,23 +13,7 @@ package raw.compiler.rql2 import org.graalvm.polyglot.{Context, PolyglotAccess, PolyglotException, Source, Value} -import raw.client.api.{ - CompilerService, - ErrorMessage, - ErrorPosition, - ErrorRange, - LocationBinarySetting, - LocationBooleanSetting, - LocationDescription, - LocationDurationSetting, - LocationIntArraySetting, - LocationIntSetting, - LocationKVSetting, - LocationSettingKey, - LocationSettingValue, - LocationStringSetting, - ProgramEnvironment -} +import raw.client.api.{CompilerService, ErrorMessage, ErrorPosition, ErrorRange, ProgramEnvironment} import raw.compiler.base.source.Type import raw.compiler.rql2.antlr4.ParserErrors import raw.compiler.rql2.api._ @@ -72,9 +56,7 @@ trait StagedCompiler { val ctxBuilder = Context .newBuilder("rql") .engine(engine) - .environment("RAW_USER", environment.user.uid.toString) - .environment("RAW_TRACE_ID", environment.user.uid.toString) - .environment("RAW_SCOPES", environment.scopes.mkString(",")) + .environment("RAW_PROGRAM_ENVIRONMENT", ProgramEnvironment.serializeToString(environment)) .allowExperimentalOptions(true) .allowPolyglotAccess(PolyglotAccess.ALL) environment.options.get("staged-compiler").foreach { stagedCompiler => @@ -138,7 +120,7 @@ trait StagedCompiler { } } - private def polyglotValueToRql2Value(v: Value, t: Type): Rql2Value = { + private def polyglotValueToRql2Value(v: Value, t: Type)(implicit settings: RawSettings): Rql2Value = { t match { case t: Rql2TypeWithProperties if t.props.contains(Rql2IsTryableTypeProperty()) => if (v.isException) { @@ -157,7 +139,6 @@ trait StagedCompiler { } else { Rql2OptionValue(Some(polyglotValueToRql2Value(v, t.cloneAndRemoveProp(Rql2IsNullableTypeProperty())))) } - case _: Rql2UndefinedType => throw new AssertionError("Rql2Undefined is not triable and is not nullable.") case _: Rql2BoolType => Rql2BoolValue(v.asBoolean()) case _: Rql2StringType => Rql2StringValue(v.asString()) @@ -191,7 +172,7 @@ trait StagedCompiler { } Rql2BinaryValue(byteArray) case Rql2RecordType(atts, _) => - val vs = atts.map(att => polyglotValueToRql2Value(v.getMember(att.idn), att.tipe)) + val vs = atts.map(att => Rql2RecordAttr(att.idn, polyglotValueToRql2Value(v.getMember(att.idn), att.tipe))) Rql2RecordValue(vs) case Rql2ListType(innerType, _) => val seq = mutable.ArrayBuffer[Rql2Value]() @@ -218,51 +199,14 @@ trait StagedCompiler { val tipe = tipes(idx) polyglotValueToRql2Value(v1, tipe) case _: Rql2LocationType => - val url = v.asString - assert(v.hasMembers); - val members = v.getMemberKeys - val settings = mutable.Map.empty[LocationSettingKey, LocationSettingValue] - val keys = members.iterator() - while (keys.hasNext) { - val key = keys.next() - val tv = v.getMember(key) - val value = - if (tv.isNumber) LocationIntSetting(tv.asInt) - else if (tv.isBoolean) LocationBooleanSetting(tv.asBoolean) - else if (tv.isString) LocationStringSetting(tv.asString) - else if (tv.hasBufferElements) { - val bufferSize = tv.getBufferSize.toInt - val byteArray = new Array[Byte](bufferSize) - for (i <- 0 until bufferSize) { - byteArray(i) = tv.readBufferByte(i) - } - LocationBinarySetting(byteArray) - } else if (tv.isDuration) LocationDurationSetting(tv.asDuration()) - else if (tv.hasArrayElements) { - // in the context of a location, it's int-array for sure - val size = tv.getArraySize - val array = new Array[Int](size.toInt) - for (i <- 0L until size) { - array(i.toInt) = tv.getArrayElement(i).asInt - } - LocationIntArraySetting(array) - } else if (tv.hasHashEntries) { - // kv settings - val iterator = tv.getHashEntriesIterator - val keyValues = mutable.ArrayBuffer.empty[(String, String)] - while (iterator.hasIteratorNextElement) { - val kv = iterator.getIteratorNextElement // array with two elements: key and value - val key = kv.getArrayElement(0).asString - val value = kv.getArrayElement(1).asString - keyValues += ((key, value)) - } - LocationKVSetting(keyValues) - } else { - throw new AssertionError("Unexpected value type: " + tv) - } - settings.put(LocationSettingKey(key), value) + val bufferSize = v.getBufferSize.toInt + val byteArray = new Array[Byte](bufferSize) + for (i <- 0 until bufferSize) { + byteArray(i) = v.readBufferByte(i) } - Rql2LocationValue(LocationDescription(url, settings.toMap)) + val location = LocationDescription.toLocation(LocationDescription.deserialize(byteArray)) + val publicDescription = v.asString() + Rql2LocationValue(location, publicDescription) } } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/EntryExtensionProvider.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/EntryExtensionProvider.scala index 0ca29e66b..e3b411edb 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/EntryExtensionProvider.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/EntryExtensionProvider.scala @@ -136,7 +136,7 @@ object EntryExtensionProvider { new raw.compiler.rql2.builtin.ContainsListEntry, new raw.compiler.rql2.builtin.ZipListEntry, new raw.compiler.rql2.builtin.MkStringListEntry, - new raw.compiler.rql2.builtin.LocationBuildEntry, + new raw.compiler.rql2.builtin.LocationFromStringEntry, new raw.compiler.rql2.builtin.LocationDescribeEntry, new raw.compiler.rql2.builtin.LocationLsEntry, new raw.compiler.rql2.builtin.LocationLlEntry, diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/LocationDescription.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/LocationDescription.scala new file mode 100644 index 000000000..78532812a --- /dev/null +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/LocationDescription.scala @@ -0,0 +1,703 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.compiler.rql2.api + +import raw.sources.api.Location +import raw.sources.bytestream.github.GitHubLocation +import raw.sources.bytestream.http.HttpByteStreamLocation +import raw.sources.bytestream.inmemory.InMemoryByteStreamLocation +import raw.sources.filesystem.api.FileSystemLocation +import raw.sources.filesystem.dropbox.{DropboxAccessTokenPath, DropboxUsernamePasswordPath} +import raw.sources.filesystem.local.LocalPath +import raw.sources.filesystem.mock.MockPath +import raw.sources.filesystem.s3.S3Path +import raw.sources.jdbc.mysql.{MySqlSchemaLocation, MySqlServerLocation, MySqlTableLocation} +import raw.sources.jdbc.oracle.{OracleSchemaLocation, OracleServerLocation, OracleTableLocation} +import raw.sources.jdbc.pgsql.{PostgresqlSchemaLocation, PostgresqlServerLocation, PostgresqlTableLocation} +import raw.sources.jdbc.snowflake.{SnowflakeSchemaLocation, SnowflakeServerLocation, SnowflakeTableLocation} +import raw.sources.jdbc.sqlite.{SqliteSchemaLocation, SqliteServerLocation, SqliteTableLocation} +import raw.sources.jdbc.sqlserver.{SqlServerSchemaLocation, SqlServerServerLocation, SqlServerTableLocation} +import raw.sources.jdbc.teradata.{TeradataSchemaLocation, TeradataServerLocation, TeradataTableLocation} +import raw.utils.RawSettings + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType} +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} +import com.typesafe.scalalogging.StrictLogging +import raw.client.api.{ + JdbcLocation, + MySqlJdbcLocation, + OracleJdbcLocation, + PostgresJdbcLocation, + ProgramEnvironment, + SnowflakeJdbcLocation, + SqlServerJdbcLocation, + SqliteJdbcLocation, + TeradataJdbcLocation +} + +import java.net.{HttpURLConnection, URI, URISyntaxException} + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes( + Array( + new JsonType(value = classOf[GitHubLocationDescription], name = "github"), + new JsonType(value = classOf[HttpByteStreamLocationDescription], name = "http"), + new JsonType(value = classOf[InMemoryByteStreamLocationDescription], name = "in-memory"), + new JsonType(value = classOf[DropboxAccessTokenLocationDescription], name = "dropbox-access-token"), + new JsonType(value = classOf[DropboxUsernamePasswordLocationDescription], name = "dropbox-username-password"), + new JsonType(value = classOf[LocalPathLocationDescription], name = "local-path"), + new JsonType(value = classOf[MockPathLocationDescription], name = "mock-path"), + new JsonType(value = classOf[S3PathLocationDescription], name = "s3"), + new JsonType(value = classOf[MySqlServerLocationDescription], name = "mysql-server"), + new JsonType(value = classOf[MySqlSchemaLocationDescription], name = "mysql-schema"), + new JsonType(value = classOf[MySqlTableLocationDescription], name = "mysql-table"), + new JsonType(value = classOf[OracleServerLocationDescription], name = "oracle-server"), + new JsonType(value = classOf[OracleSchemaLocationDescription], name = "oracle-schema"), + new JsonType(value = classOf[OracleTableLocationDescription], name = "oracle-table"), + new JsonType(value = classOf[PostgresqlServerLocationDescription], name = "postgresql-server"), + new JsonType(value = classOf[PostgresqlSchemaLocationDescription], name = "postgresql-schema"), + new JsonType(value = classOf[PostgresqlTableLocationDescription], name = "postgresql-table"), + new JsonType(value = classOf[SnowflakeServerLocationDescription], name = "snowflake-server"), + new JsonType(value = classOf[SnowflakeSchemaLocationDescription], name = "snowflake-schema"), + new JsonType(value = classOf[SnowflakeTableLocationDescription], name = "snowflake-table"), + new JsonType(value = classOf[SqliteServerLocationDescription], name = "sqlite-server"), + new JsonType(value = classOf[SqliteSchemaLocationDescription], name = "sqlite-schema"), + new JsonType(value = classOf[SqliteTableLocationDescription], name = "sqlite-table"), + new JsonType(value = classOf[SqlServerServerLocationDescription], name = "sqlserver-server"), + new JsonType(value = classOf[SqlServerSchemaLocationDescription], name = "sqlserver-schema"), + new JsonType(value = classOf[SqlServerTableLocationDescription], name = "sqlserver-table"), + new JsonType(value = classOf[TeradataServerLocationDescription], name = "teradata-server"), + new JsonType(value = classOf[TeradataSchemaLocationDescription], name = "teradata-schema"), + new JsonType(value = classOf[TeradataTableLocationDescription], name = "teradata-table") + ) +) +sealed trait LocationDescription +final case class GitHubLocationDescription(username: String, repo: String, file: String, maybeBranch: Option[String]) + extends LocationDescription +final case class HttpByteStreamLocationDescription( + url: String, + method: String, + args: Array[(String, String)], + headers: Array[(String, String)], + maybeBody: Option[Array[Byte]], + expectedStatus: Array[Int] +) extends LocationDescription +final case class InMemoryByteStreamLocationDescription(data: Array[Byte]) extends LocationDescription +final case class DropboxAccessTokenLocationDescription(accessToken: String, path: String) extends LocationDescription +final case class DropboxUsernamePasswordLocationDescription(username: String, password: String, path: String) + extends LocationDescription +final case class LocalPathLocationDescription(path: String) extends LocationDescription +final case class MockPathLocationDescription(delayMillis: Long, delegate: LocationDescription) + extends LocationDescription +final case class S3PathLocationDescription( + bucket: String, + maybeRegion: Option[String], + maybeAccessKey: Option[String], + maybeSecretKey: Option[String], + path: String +) extends LocationDescription +final case class MySqlServerLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String +) extends LocationDescription +final case class MySqlSchemaLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String +) extends LocationDescription +final case class MySqlTableLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + tableName: String +) extends LocationDescription +final case class OracleServerLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String +) extends LocationDescription +final case class OracleSchemaLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String +) extends LocationDescription +final case class OracleTableLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String, + tableName: String +) extends LocationDescription +final case class PostgresqlServerLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String +) extends LocationDescription +final case class PostgresqlSchemaLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String +) extends LocationDescription +final case class PostgresqlTableLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String, + tableName: String +) extends LocationDescription +final case class SnowflakeServerLocationDescription( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String] +) extends LocationDescription +final case class SnowflakeSchemaLocationDescription( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String], + schemaName: String +) extends LocationDescription +final case class SnowflakeTableLocationDescription( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String], + schemaName: String, + tableName: String +) extends LocationDescription +final case class SqliteServerLocationDescription( + path: String +) extends LocationDescription +final case class SqliteSchemaLocationDescription( + path: String +) extends LocationDescription +final case class SqliteTableLocationDescription( + path: String, + tableName: String +) extends LocationDescription +final case class SqlServerServerLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String +) extends LocationDescription +final case class SqlServerSchemaLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String +) extends LocationDescription +final case class SqlServerTableLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String, + tableName: String +) extends LocationDescription +final case class TeradataServerLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + parameters: Map[String, String] +) extends LocationDescription +final case class TeradataSchemaLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String, + parameters: Map[String, String] +) extends LocationDescription +final case class TeradataTableLocationDescription( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schemaName: String, + tableName: String, + parameters: Map[String, String] +) extends LocationDescription + +object LocationDescription extends StrictLogging { + + val DROPBOX_REGEX = "dropbox:(?://([^/]+)?)?(.*)".r + + private val jsonMapper = new ObjectMapper with ClassTagExtensions { + registerModule(DefaultScalaModule) + registerModule(new JavaTimeModule()) + registerModule(new Jdk8Module()) + } + + private val reader = jsonMapper.readerFor[LocationDescription] + private val writer = jsonMapper.writerFor[LocationDescription] + + def toLocationDescription(l: JdbcLocation): LocationDescription = { + l match { + case MySqlJdbcLocation(host, port, database, username, password) => + MySqlServerLocationDescription(host, port, database, username, password) + case OracleJdbcLocation(host, port, database, username, password) => + OracleServerLocationDescription(host, port, database, username, password) + case PostgresJdbcLocation(host, port, database, username, password) => + PostgresqlServerLocationDescription(host, port, database, username, password) + case SnowflakeJdbcLocation(database, username, password, accountIdentifier, parameters) => + SnowflakeServerLocationDescription(database, username, password, accountIdentifier, parameters) + case SqliteJdbcLocation(path) => SqliteServerLocationDescription(path) + case SqlServerJdbcLocation(host, port, database, username, password) => + SqlServerServerLocationDescription(host, port, database, username, password) + case TeradataJdbcLocation(host, port, database, username, password, parameters) => + TeradataServerLocationDescription(host, port, database, username, password, parameters) + } + } + + def toLocationDescription(l: Location): LocationDescription = { + l match { + case g: GitHubLocation => GitHubLocationDescription(g.username, g.repo, g.file, g.maybeBranch) + case h: HttpByteStreamLocation => + HttpByteStreamLocationDescription(h.url, h.method, h.args, h.headers, h.maybeBody, h.expectedStatus) + case i: InMemoryByteStreamLocation => InMemoryByteStreamLocationDescription(i.data) + case d: DropboxAccessTokenPath => DropboxAccessTokenLocationDescription(d.accessToken, d.path) + case d: DropboxUsernamePasswordPath => DropboxUsernamePasswordLocationDescription(d.username, d.password, d.path) + case l: LocalPath => LocalPathLocationDescription(l.pathName) + case m: MockPath => MockPathLocationDescription(m.delayMillis, toLocationDescription(m.delegate)) + case s: S3Path => S3PathLocationDescription(s.bucket, s.region, s.maybeAccessKey, s.maybeSecretKey, s.path) + case m: MySqlServerLocation => MySqlServerLocationDescription(m.host, m.port, m.dbName, m.username, m.password) + case m: MySqlSchemaLocation => MySqlSchemaLocationDescription(m.host, m.port, m.dbName, m.username, m.password) + case m: MySqlTableLocation => + MySqlTableLocationDescription(m.host, m.port, m.dbName, m.username, m.password, m.table) + case o: OracleServerLocation => OracleServerLocationDescription(o.host, o.port, o.dbName, o.username, o.password) + case o: OracleSchemaLocation => + OracleSchemaLocationDescription(o.host, o.port, o.dbName, o.username, o.password, o.schema) + case o: OracleTableLocation => + OracleTableLocationDescription(o.host, o.port, o.dbName, o.username, o.password, o.schema, o.table) + case p: PostgresqlServerLocation => + PostgresqlServerLocationDescription(p.host, p.port, p.dbName, p.username, p.password) + case p: PostgresqlSchemaLocation => + PostgresqlSchemaLocationDescription(p.host, p.port, p.dbName, p.username, p.password, p.schema) + case p: PostgresqlTableLocation => + PostgresqlTableLocationDescription(p.host, p.port, p.dbName, p.username, p.password, p.schema, p.table) + case s: SnowflakeServerLocation => + SnowflakeServerLocationDescription(s.dbName, s.username, s.password, s.accountIdentifier, s.parameters) + case s: SnowflakeSchemaLocation => SnowflakeSchemaLocationDescription( + s.dbName, + s.username, + s.password, + s.accountIdentifier, + s.parameters, + s.schema + ) + case s: SnowflakeTableLocation => SnowflakeTableLocationDescription( + s.dbName, + s.username, + s.password, + s.accountIdentifier, + s.parameters, + s.schema, + s.table + ) + case s: SqliteServerLocation => SqliteServerLocationDescription(s.path) + case s: SqliteSchemaLocation => SqliteSchemaLocationDescription(s.path) + case s: SqliteTableLocation => SqliteTableLocationDescription(s.path, s.table) + case s: SqlServerServerLocation => + SqlServerServerLocationDescription(s.host, s.port, s.dbName, s.username, s.password) + case s: SqlServerSchemaLocation => + SqlServerSchemaLocationDescription(s.host, s.port, s.dbName, s.username, s.password, s.schema) + case s: SqlServerTableLocation => + SqlServerTableLocationDescription(s.host, s.port, s.dbName, s.username, s.password, s.schema, s.table) + case t: TeradataServerLocation => + TeradataServerLocationDescription(t.host, t.port, t.dbName, t.username, t.password, t.parameters) + case t: TeradataSchemaLocation => + TeradataSchemaLocationDescription(t.host, t.port, t.dbName, t.username, t.password, t.schema, t.parameters) + case t: TeradataTableLocation => TeradataTableLocationDescription( + t.host, + t.port, + t.dbName, + t.username, + t.password, + t.schema, + t.table, + t.parameters + ) + } + } + + def toLocation(l: JdbcLocation)(implicit settings: RawSettings): Location = { + toLocation(toLocationDescription(l)) + } + + def toLocation(l: LocationDescription)(implicit settings: RawSettings): Location = { + l match { + case GitHubLocationDescription(username, repo, file, maybeBranch) => + new GitHubLocation(username, repo, file, maybeBranch) + case HttpByteStreamLocationDescription(url, method, args, headers, maybeBody, expectedStatus) => + new HttpByteStreamLocation(url, method, args, headers, maybeBody, expectedStatus) + case InMemoryByteStreamLocationDescription(data) => new InMemoryByteStreamLocation(data) + case DropboxAccessTokenLocationDescription(accessToken, path) => new DropboxAccessTokenPath(accessToken, path) + case DropboxUsernamePasswordLocationDescription(username, password, path) => + new DropboxUsernamePasswordPath(username, password, path) + case LocalPathLocationDescription(path) => new LocalPath(path) + case MockPathLocationDescription(delayMillis, delegate) => + new MockPath(delayMillis, toLocation(delegate).asInstanceOf[FileSystemLocation]) + case S3PathLocationDescription(bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path) => + new S3Path(bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path) + case MySqlServerLocationDescription(host, port, dbName, username, password) => + new MySqlServerLocation(host, port, dbName, username, password) + case MySqlSchemaLocationDescription(host, port, dbName, username, password) => + new MySqlSchemaLocation(host, port, dbName, username, password) + case MySqlTableLocationDescription(host, port, dbName, username, password, tableName) => + new MySqlTableLocation(host, port, dbName, username, password, tableName) + case OracleServerLocationDescription(host, port, dbName, username, password) => + new OracleServerLocation(host, port, dbName, username, password) + case OracleSchemaLocationDescription(host, port, dbName, username, password, schemaName) => + new OracleSchemaLocation(host, port, dbName, username, password, schemaName) + case OracleTableLocationDescription(host, port, dbName, username, password, schemaName, tableName) => + new OracleTableLocation(host, port, dbName, username, password, schemaName, tableName) + case SnowflakeServerLocationDescription(dbName, username, password, accountIdentifier, parameters) => + new SnowflakeServerLocation(dbName, username, password, accountIdentifier, parameters) + case SnowflakeSchemaLocationDescription(dbName, username, password, accountIdentifier, parameters, schemaName) => + new SnowflakeSchemaLocation(dbName, username, password, accountIdentifier, parameters, schemaName) + case SnowflakeTableLocationDescription( + dbName, + username, + password, + accountIdentifier, + parameters, + schemaName, + tableName + ) => + new SnowflakeTableLocation(dbName, username, password, accountIdentifier, parameters, schemaName, tableName) + case SqliteServerLocationDescription(path) => new SqliteServerLocation(path) + case SqliteSchemaLocationDescription(path) => new SqliteSchemaLocation(path) + case SqliteTableLocationDescription(path, tableName) => new SqliteTableLocation(path, tableName) + case SqlServerServerLocationDescription(host, port, dbName, username, password) => + new SqlServerServerLocation(host, port, dbName, username, password) + case SqlServerSchemaLocationDescription(host, port, dbName, username, password, schemaName) => + new SqlServerSchemaLocation(host, port, dbName, username, password, schemaName) + case SqlServerTableLocationDescription(host, port, dbName, username, password, schemaName, tableName) => + new SqlServerTableLocation(host, port, dbName, username, password, schemaName, tableName) + case TeradataServerLocationDescription(host, port, dbName, username, password, parameters) => + new TeradataServerLocation(host, port, dbName, username, password, parameters) + case TeradataSchemaLocationDescription(host, port, dbName, username, password, schemaName, parameters) => + new TeradataSchemaLocation(host, port, dbName, username, password, schemaName, parameters) + case TeradataTableLocationDescription( + host, + port, + dbName, + username, + password, + schemaName, + tableName, + parameters + ) => new TeradataTableLocation(host, port, dbName, username, password, schemaName, tableName, parameters) + } + } + + def toLocation(bytes: Array[Byte])(implicit settings: RawSettings): Location = { + toLocation(deserialize(bytes)) + } + + def serialize(l: LocationDescription): Array[Byte] = { + val output = new ByteArrayOutputStream() + try { + val generator = jsonMapper.getFactory.createGenerator(output) + writer.writeValue(generator, l) + generator.flush() + output.toByteArray + } finally { + output.close() + } + } + + def deserialize(bytes: Array[Byte]): LocationDescription = { + reader.readValue(new ByteArrayInputStream(bytes)) + } + + def locationDescriptionToPublicUrl(l: LocationDescription): String = { + l match { + case GitHubLocationDescription(username, repo, file, maybeBranch) => maybeBranch match { + case Some(branch) => s"github://$repo/$username/$file?branch=$branch" + case None => s"github://$repo/$username/$file" + } + case HttpByteStreamLocationDescription(url, _, _, _, _, _) => url + case DropboxAccessTokenLocationDescription(_, path) => s"dropbox:$path" + case DropboxUsernamePasswordLocationDescription(_, _, path) => s"dropbox:$path" + case LocalPathLocationDescription(path) => + // TODO (msb): We should move to file:// + s"file:$path" + case MockPathLocationDescription(_, delegate) => + // TODO (msb): We should move to mock:// + s"mock:${locationDescriptionToPublicUrl(delegate)}" + case S3PathLocationDescription(bucket, _, _, _, path) => s"s3://$bucket/$path" + case MySqlServerLocationDescription(host, port, dbName, _, _) => s"mysql://$host:$port/$dbName" + case MySqlSchemaLocationDescription(host, port, dbName, _, _) => s"mysql://$host:$port/$dbName" + case MySqlTableLocationDescription(host, port, dbName, _, _, tableName) => + s"mysql://$host:$port/$dbName/$tableName" + case OracleServerLocationDescription(host, port, dbName, _, _) => s"oracle://$host:$port/$dbName" + case OracleSchemaLocationDescription(host, port, dbName, _, _, schemaName) => + s"oracle://$host:$port/$dbName/$schemaName" + case OracleTableLocationDescription(host, port, dbName, _, _, schemaName, tableName) => + s"oracle://$host:$port/$dbName/$schemaName/$tableName" + case PostgresqlServerLocationDescription(host, port, dbName, _, _) => s"pgsql://$host:$port/$dbName" + case PostgresqlSchemaLocationDescription(host, port, dbName, _, _, schemaName) => + s"pgsql://$host:$port/$dbName/$schemaName" + case PostgresqlTableLocationDescription(host, port, dbName, _, _, schemaName, tableName) => + s"pgsql://$host:$port/$dbName/$schemaName/$tableName" + case SnowflakeServerLocationDescription(dbName, _, _, _, _) => s"snowflake://$dbName" + case SnowflakeSchemaLocationDescription(dbName, _, _, _, _, schemaName) => s"snowflake://$dbName/$schemaName" + case SnowflakeTableLocationDescription(dbName, _, _, _, _, schemaName, tableName) => + s"snowflake://$dbName/$schemaName/$tableName" + case SqliteServerLocationDescription(path) => s"sqlite://$path" + case SqliteSchemaLocationDescription(path) => s"sqlite://$path" + case SqliteTableLocationDescription(path, tableName) => s"sqlite://$path/$tableName" + case SqlServerServerLocationDescription(host, port, dbName, _, _) => s"sqlserver://$host:$port/$dbName" + case SqlServerSchemaLocationDescription(host, port, dbName, _, _, schemaName) => + s"sqlserver://$host:$port/$dbName/$schemaName" + case SqlServerTableLocationDescription(host, port, dbName, _, _, schemaName, tableName) => + s"sqlserver://$host:$port/$dbName/$schemaName/$tableName" + case TeradataServerLocationDescription(host, port, dbName, _, _, _) => s"teradata://$host:$port/$dbName" + case TeradataSchemaLocationDescription(host, port, dbName, _, _, schemaName, _) => + s"teradata://$host:$port/$dbName/$schemaName" + case TeradataTableLocationDescription(host, port, dbName, _, _, schemaName, tableName, _) => + s"teradata://$host:$port/$dbName/$schemaName/$tableName" + } + } + + def locationToPublicUrl(l: Location): String = { + locationDescriptionToPublicUrl(toLocationDescription(l)) + } + + def urlToLocationDescription(url: String, programEnvironment: ProgramEnvironment)( + implicit settings: RawSettings + ): Either[String, LocationDescription] = { + // Extract the protocol. + val colonIndex = url.indexOf(':') + if (colonIndex == -1) { + return Left(s"missing protocol: $url") + } + val protocol = url.substring(0, colonIndex) + + // Parse the URL based on the protocol. + protocol match { + case "http" | "https" => + // FIXME: This is ignoring query parameters!!! + Right( + HttpByteStreamLocationDescription( + url, + method = "GET", + args = Array.empty, + headers = Array.empty, + maybeBody = None, + expectedStatus = Array( + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_ACCEPTED, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_PARTIAL + ) + ) + ) + case "file" if settings.onTrainingWheels => Right(LocalPathLocationDescription(url.substring(colonIndex + 1))) + case "s3" => + // Build a URI to validate the URL. + val uri = { + try { + new URI(url) + } catch { + case _: URISyntaxException => return Left("invalid S3 URL: " + url) + } + } + + val uriUserInfo = uri.getUserInfo + val bucketName = uri.getHost + val path = uri.getPath + val objectKey = if (path.startsWith("/")) path.substring(1) else path + + var maybeAccessKey: Option[String] = None + var maybeSecretKey: Option[String] = None + if (uriUserInfo != null) { + val userInfoParts = uriUserInfo.split(":") + maybeAccessKey = Some(userInfoParts(0)) + if (maybeAccessKey.get.isEmpty) { + return Left("missing S3 access key") + } + if (userInfoParts.length > 1) { + maybeSecretKey = Some(userInfoParts(1)) + if (maybeSecretKey.get.isEmpty) { + return Left("missing S3 secret key") + } + } else { + return Left("missing S3 secret key") + } + } + + if (maybeAccessKey.isEmpty) { + // If the access key/secret key are not defined, check if credential exists. + programEnvironment.s3Credentials.get(bucketName) match { + case Some(s3Credential) => Right( + S3PathLocationDescription( + bucketName, + s3Credential.region, + s3Credential.accessKey, + s3Credential.secretKey, + objectKey + ) + ) + case None => + // Anonymous access. + Right(S3PathLocationDescription(bucketName, None, None, None, objectKey)) + } + } else { + // TODO (msb): There is no way to specify the region when using a direct URL... + Right(S3PathLocationDescription(bucketName, None, maybeAccessKey, maybeSecretKey, objectKey)) + } + case "dropbox" => + // In Dropbox, the host is the name of the credential + val DROPBOX_REGEX(name, path) = url + if (name == null) { + return Left("missing Dropbox credential") + } + programEnvironment.httpHeaders.get(name) match { + case Some(httpHeaders) => Right( + DropboxAccessTokenLocationDescription( + httpHeaders("Authorization").split("Bearer ")(1), + path + ) + ) + case None => Left("missing Dropbox credential") + } + case _ => Left(s"unsupported protocol: $protocol") + } + } + + // @CompilerDirectives.TruffleBoundary +// private Location getPgsqlLocation(String url, RawContext context) { +// try { +// URI uri = new URI(url); +// String uriUserInfo = uri.getUserInfo(); +// String uriHost = uri.getHost(); +// int uriPort = uri.getPort(); +// String uriPath = uri.getPath(); +// +// String host = null; +// Integer port = null; +// String username = null; +// String password = null; +// String dbname = null; +// String schema = null; +// String table = null; +// +// if (uriUserInfo != null) { +// String[] userInfoParts = uriUserInfo.split(":"); +// username = userInfoParts[0]; +// if (username.isEmpty()) { +// throw new RawTruffleRuntimeException("missing PostgreSQL username"); +// } +// if (userInfoParts.length > 1) { +// password = userInfoParts[1]; +// if (password.isEmpty()) { +// throw new RawTruffleRuntimeException("missing PostgreSQL password"); +// } +// } else { +// throw new RawTruffleRuntimeException("missing PostgreSQL password"); +// } +// } +// +// if (username == null) { +// // If the username/password are not defined, then the "host" is actually the database name +// // in the program environment credentials set. +// JdbcLocation jdbcLocation = +// RawContext.get(this).getProgramEnvironment().jdbcServers().get(uriHost).get(); +// if (jdbcLocation instanceof PostgresJdbcLocation) { +// PostgresJdbcLocation pgsqlLocation = (PostgresJdbcLocation) jdbcLocation; +// host = pgsqlLocation.host(); +// port = pgsqlLocation.port(); +// username = pgsqlLocation.username(); +// password = pgsqlLocation.password(); +// } else { +// throw new RawTruffleRuntimeException("not a PostgreSQL credential: " + uriHost); +// } +// } else { +// // If the username/password are defined, then the "host" is the actual host. +// +// host = uriHost; +// port = uriPort; +// +// if (uriPath != null) { +// String[] pathParts = uriPath.split("/"); +// dbname = pathParts.length > 1 ? pathParts[1] : null; +// schema = pathParts.length > 2 ? pathParts[2] : null; +// table = pathParts.length > 3 ? pathParts[3] : null; +// } +// } +// +// if (dbname != null && schema != null && table != null) { +// return new PostgresqlTableLocation( +// host, port, dbname, username, password, schema, table, context.getSettings()); +// } else if (dbname != null && schema != null) { +// return new PostgresqlSchemaLocation( +// host, port, dbname, username, password, schema, context.getSettings()); +// } else if (dbname != null) { +// return new PostgresqlServerLocation(host, port, dbname, username, password, context.getSettings()); +// } else { +// throw new RawTruffleRuntimeException("invalid PostgreSQL URL: " + url); +// } +// +// } catch (URISyntaxException e) { +// throw new RawTruffleRuntimeException("invalid PostgreSQL URL: " + url); +// } +// } + + def urlToLocation(url: String, programEnvironment: ProgramEnvironment)( + implicit settings: RawSettings + ): Either[String, Location] = { + urlToLocationDescription(url, programEnvironment).right.map(toLocation) + } + +} diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/PackageExtension.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/PackageExtension.scala index da02778d2..610cda5e2 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/PackageExtension.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/PackageExtension.scala @@ -12,14 +12,14 @@ package raw.compiler.rql2.api -import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic, UnsupportedType} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ -import raw.compiler.rql2.builtin.{ListPackageBuilder, LocationPackageBuilder, RecordPackageBuilder} import raw.compiler.rql2.source._ import raw.compiler.rql2.{ProgramContext, Rql2TypeUtils} import raw.client.api._ import raw.sources.api._ +import raw.sources.bytestream.api.ByteStreamLocation import scala.annotation.nowarn import scala.collection.immutable.ListMap @@ -172,7 +172,7 @@ abstract class ShortEntryExtension( trait EntryExtensionHelper extends Rql2TypeUtils { /////////////////////////////////////////////////////////////////////////// - // Helpers + // Value Helpers /////////////////////////////////////////////////////////////////////////// final protected def getStringValue(v: Arg): String = { v.asInstanceOf[ValueArg].v.asInstanceOf[Rql2StringValue].v } @@ -181,31 +181,19 @@ trait EntryExtensionHelper extends Rql2TypeUtils { final protected def getBoolValue(v: Arg): Boolean = { v.asInstanceOf[ValueArg].v.asInstanceOf[Rql2BoolValue].v } - final protected def getLocationValue(v: Arg): LocationDescription = { - v.asInstanceOf[ValueArg].v.asInstanceOf[Rql2LocationValue].v + final protected def getByteStreamLocation(v: Arg): Either[String, ByteStreamLocation] = { + val locationValue = v.asInstanceOf[ValueArg].v.asInstanceOf[Rql2LocationValue] + locationValue.l match { + case l: ByteStreamLocation => Right(l) + case _ => Left("expected a bytestream") + } } final protected def locationValueToExp(v: Arg): Exp = { - val description = getLocationValue(v) - LocationPackageBuilder.Build( - StringConst(description.url), - description.settings.map { - case (k, v) => - val idn = k.key - val exp = v match { - case LocationIntSetting(value) => IntConst(value.toString) - case LocationStringSetting(value) => StringConst(value) - case LocationBinarySetting(bytes) => BinaryConst(bytes.toArray) - case LocationBooleanSetting(value) => BoolConst(value) - case LocationDurationSetting(value) => ??? - case LocationKVSetting(settings) => ListPackageBuilder.Build(settings.map { - case (k, v) => RecordPackageBuilder.Build(StringConst(k), StringConst(v)) - }: _*) - case LocationIntArraySetting(value) => ListPackageBuilder.Build(value.map(i => IntConst(i.toString)): _*) - } - (idn, exp) - }.toVector - ) + val locationValue = v.asInstanceOf[ValueArg].v.asInstanceOf[Rql2LocationValue] + val location = locationValue.l + val locationDescription = LocationDescription.toLocationDescription(location) + LocationConst(LocationDescription.serialize(locationDescription), locationValue.publicDescription) } final protected def getListStringValue(v: Arg): Seq[String] = { @@ -229,9 +217,9 @@ trait EntryExtensionHelper extends Rql2TypeUtils { .v .map { x => val values = x.asInstanceOf[Rql2RecordValue].v.map { - case Rql2OptionValue(Some(v: Rql2StringValue)) => Some(v.v) - case Rql2StringValue(v) => Some(v) - case Rql2OptionValue(None) => None + case Rql2RecordAttr(_, Rql2OptionValue(Some(v: Rql2StringValue))) => Some(v.v) + case Rql2RecordAttr(_, Rql2StringValue(v)) => Some(v) + case Rql2RecordAttr(_, Rql2OptionValue(None)) => None } (values(0), values(1)) } @@ -256,6 +244,31 @@ trait EntryExtensionHelper extends Rql2TypeUtils { varArgs(idx).asInstanceOf[ExpArg].e } + /////////////////////////////////////////////////////////////////////////// + // Validator Helpers + /////////////////////////////////////////////////////////////////////////// + + protected def validateTableType(t: Type): Either[Seq[UnsupportedType], Rql2IterableType] = t match { + case Rql2IterableType(Rql2RecordType(atts, _), _) => + val validated = atts.map { x => + x.tipe match { + case _: Rql2StringType => Right(x) + case _: Rql2BoolType => Right(x) + case _: Rql2NumberType => Right(x) + case _: Rql2DateType => Right(x) + case _: Rql2TimeType => Right(x) + case _: Rql2TimestampType => Right(x) + case _: Rql2BinaryType => Right(x) + // intervals are not supported, so we cannot match temporal types here. + case _ => Left(Seq(UnsupportedType(x.tipe, x.tipe, None))) + } + } + val errors = validated.collect { case Left(error) => error } + if (errors.nonEmpty) Left(errors.flatten) + else Right(Rql2IterableType(Rql2RecordType(atts))) + case _ => Left(Seq(UnsupportedType(t, t, None))) + } + /////////////////////////////////////////////////////////////////////////// // Type Helpers /////////////////////////////////////////////////////////////////////////// diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/Values.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/Values.scala index a5fe3c959..cce0c35f1 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/api/Values.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/api/Values.scala @@ -12,7 +12,7 @@ package raw.compiler.rql2.api -import raw.client.api.LocationDescription +import raw.sources.api.Location sealed trait Rql2Value final case class Rql2StringValue(v: String) extends Rql2Value @@ -38,10 +38,10 @@ final case class Rql2IntervalValue( seconds: Int, millis: Int ) extends Rql2Value -final case class Rql2LocationValue(v: LocationDescription) extends Rql2Value -final case class Rql2RecordValue(v: Seq[Rql2Value]) extends Rql2Value +final case class Rql2LocationValue(l: Location, publicDescription: String) extends Rql2Value +final case class Rql2RecordValue(v: Seq[Rql2RecordAttr]) extends Rql2Value +final case class Rql2RecordAttr(name: String, value: Rql2Value) final case class Rql2TryValue(v: Either[String, Rql2Value]) extends Rql2Value final case class Rql2OptionValue(v: Option[Rql2Value]) extends Rql2Value final case class Rql2ListValue(v: Seq[Rql2Value]) extends Rql2Value final case class Rql2IterableValue(v: Seq[Rql2Value]) extends Rql2Value // Data has been ready is now materialized. -final case class Rql2OrValue(vs: Seq[Rql2Value]) extends Rql2Value diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/CsvPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/CsvPackage.scala index 6d3e1c4c5..1891e0f96 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/CsvPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/CsvPackage.scala @@ -23,13 +23,16 @@ import raw.compiler.rql2.api.{ ExpParam, PackageExtension, Param, + Rql2LocationValue, SugarEntryExtension, TypeParam, + ValueArg, ValueParam } import raw.compiler.rql2.source._ import raw.client.api._ import raw.inferrer.api._ +import raw.sources.bytestream.inmemory.InMemoryByteStreamLocation class CsvPackage extends PackageExtension { @@ -303,7 +306,6 @@ class CsvInferAndReadEntry extends SugarEntryExtension with CsvEntryExtensionHel Proj(PackageIdnExp("Csv"), "Read"), l0Args ) - } } @@ -584,11 +586,12 @@ class CsvInferAndParseEntry extends SugarEntryExtension with CsvEntryExtensionHe optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { - - val (locationArg, _) = InMemoryLocationValueBuilder.build(mandatoryArgs) - + val codeData = getStringValue(mandatoryArgs.head) for ( - inferrerProperties <- getCsvInferrerProperties(Seq(locationArg), optionalArgs); + inferrerProperties <- getCsvInferrerProperties( + Seq(ValueArg(Rql2LocationValue(new InMemoryByteStreamLocation(codeData), ""), Rql2LocationType())), + optionalArgs + ); inputFormatDescriptor <- programContext.infer(inferrerProperties); TextInputStreamFormatDescriptor( _, @@ -629,11 +632,13 @@ class CsvInferAndParseEntry extends SugarEntryExtension with CsvEntryExtensionHe optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Exp = { - - val (locationArg, codeData) = InMemoryLocationValueBuilder.build(mandatoryArgs) + val codeData = getStringValue(mandatoryArgs.head) val r = for ( - inferrerProperties <- getCsvInferrerProperties(Seq(locationArg), optionalArgs); + inferrerProperties <- getCsvInferrerProperties( + Seq(ValueArg(Rql2LocationValue(new InMemoryByteStreamLocation(codeData), ""), Rql2LocationType())), + optionalArgs + ); inputFormatDescriptor <- programContext.infer(inferrerProperties) ) yield { inputFormatDescriptor @@ -832,9 +837,9 @@ trait CsvEntryExtensionHelper extends EntryExtensionHelper { mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)] ): Either[String, CsvInferrerProperties] = { - Right( + getByteStreamLocation(mandatoryArgs.head).right.map { location => CsvInferrerProperties( - getLocationValue(mandatoryArgs.head), + location, optionalArgs.collectFirst { case a if a._1 == "sampleSize" => a._2 }.map(getIntValue), optionalArgs .collectFirst { case a if a._1 == "encoding" => a._2 } @@ -851,7 +856,7 @@ trait CsvEntryExtensionHelper extends EntryExtensionHelper { .collectFirst { case a if a._1 == "quotes" => a._2 } .map(v => getListOptionStringValue(v).map(_.map(_.head))) ) - ) + } } protected def validateCsvType(t: Type): Either[Seq[UnsupportedType], Rql2IterableType] = { diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/HttpPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/HttpPackage.scala index 012e0e95e..9dc222e4c 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/HttpPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/HttpPackage.scala @@ -14,7 +14,6 @@ package raw.compiler.rql2.builtin import raw.client.api._ import raw.compiler.base.source.Type -import raw.compiler.common.source._ import raw.compiler.rql2._ import raw.compiler.rql2.api.{Arg, EntryExtension, ExpParam, PackageExtension, Param, ShortEntryExtension} import raw.compiler.rql2.source._ @@ -123,48 +122,12 @@ abstract class HttpCallEntry(method: String) extends EntryExtension { "The data to send as the body of the request. Cannot be used with `bodyString`.", isOptional = true ), - ParamDoc( - "token", - TypeDoc(List("string")), - "The bearer token to be passed as the Authorization header of the request.", - isOptional = true - ), ParamDoc( "authCredentialName", TypeDoc(List("string")), "The name of the HTTP credential registered in the credentials storage.", isOptional = true ), - ParamDoc( - "clientId", - TypeDoc(List("string")), - "The client ID to use for the client credentials OAuth flow. Requires `clientSecret` and `tokenUrl`.", - isOptional = true - ), - ParamDoc( - "clientSecret", - TypeDoc(List("string")), - "The client secret to use for the client credentials OAuth flow. Requires `clientId` and either `authProvider` or `tokenUrl`.", - isOptional = true - ), - ParamDoc( - "authProvider", - TypeDoc(List("string")), - "The provider for client ID client secret OAuth flow. Requires `clientId` and `clientSecret`.", - isOptional = true - ), - ParamDoc( - "tokenUrl", - TypeDoc(List("string")), - "The URL to be used for the client credentials OAuth flow. Requires `clientId` and `clientSecret`.", - isOptional = true - ), - ParamDoc( - "useBasicAuth", - TypeDoc(List("bool")), - "If true, uses basic auth for the client credentials OAuth flow. Requires `clientId`, `clientSecret` and `tokenUrl`.", - isOptional = true - ), ParamDoc( "username", TypeDoc(List("string")), @@ -208,13 +171,7 @@ abstract class HttpCallEntry(method: String) extends EntryExtension { Set( "bodyString", "bodyBinary", - "token", "authCredentialName", - "clientId", - "clientSecret", - "authProvider", - "tokenUrl", - "useBasicAuth", "username", "password", "args", @@ -224,26 +181,38 @@ abstract class HttpCallEntry(method: String) extends EntryExtension { ) override def getOptionalParam(prevMandatoryArgs: Seq[Arg], idn: String): Either[String, Param] = { - Right( - ExpParam( - OneOfType( - Rql2IntType(), - Rql2StringType(), - Rql2BinaryType(), - Rql2BoolType(), - Rql2IntervalType(), - Rql2ListType( - Rql2RecordType( - Vector( - Rql2AttrType("_1", Rql2StringType(Set(Rql2IsNullableTypeProperty()))), - Rql2AttrType("_2", Rql2StringType(Set(Rql2IsNullableTypeProperty()))) + idn match { + case "bodyString" => Right(ExpParam(Rql2StringType())) + case "bodyBinary" => Right(ExpParam(Rql2BinaryType())) + case "authCredentialName" => Right(ExpParam(Rql2StringType())) + case "username" => Right(ExpParam(Rql2StringType())) + case "password" => Right(ExpParam(Rql2StringType())) + case "args" => Right( + ExpParam( + Rql2ListType( + Rql2RecordType( + Vector( + Rql2AttrType("_1", Rql2StringType(Set(Rql2IsNullableTypeProperty()))), + Rql2AttrType("_2", Rql2StringType(Set(Rql2IsNullableTypeProperty()))) + ) ) ) - ), - Rql2ListType(Rql2IntType()) + ) ) - ) - ) + case "headers" => Right( + ExpParam( + Rql2ListType( + Rql2RecordType( + Vector( + Rql2AttrType("_1", Rql2StringType(Set(Rql2IsNullableTypeProperty()))), + Rql2AttrType("_2", Rql2StringType(Set(Rql2IsNullableTypeProperty()))) + ) + ) + ) + ) + ) + case "expectedStatus" => Right(ExpParam(Rql2ListType(Rql2IntType()))) + } } override def returnType( diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/InMemoryLocationValueBuilder.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/InMemoryLocationValueBuilder.scala deleted file mode 100644 index 36f65a769..000000000 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/InMemoryLocationValueBuilder.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.builtin - -import raw.compiler.rql2.api.{Arg, Rql2LocationValue, Rql2StringValue, ValueArg} -import raw.sources.bytestream.in_memory.InMemoryByteStreamLocation -import raw.client.api._ -import raw.compiler.rql2.source.Rql2LocationType - -object InMemoryLocationValueBuilder { - def build(mandatoryArgs: Seq[Arg]): (ValueArg, String) = { - val codeData = mandatoryArgs.head match { - case ValueArg(v, _) => v match { - case Rql2StringValue(innVal) => innVal - } - } - val settings = Map[LocationSettingKey, LocationSettingValue]( - ( - LocationSettingKey(InMemoryByteStreamLocation.codeDataKey), - LocationBinarySetting(codeData.getBytes()) - ) - ) - val locationDescription = LocationDescription(InMemoryByteStreamLocation.schemaWithColon, settings) - (ValueArg(Rql2LocationValue(locationDescription), Rql2LocationType()), codeData) - } -} diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/JsonPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/JsonPackage.scala index 3c42c47b6..3be61a6b9 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/JsonPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/JsonPackage.scala @@ -24,13 +24,16 @@ import raw.compiler.rql2.api.{ ExpParam, PackageExtension, Param, + Rql2LocationValue, SugarEntryExtension, TypeParam, + ValueArg, ValueParam } import raw.compiler.rql2.source._ import raw.client.api._ import raw.inferrer.api._ +import raw.sources.bytestream.inmemory.InMemoryByteStreamLocation class JsonPackage extends PackageExtension { @@ -359,13 +362,15 @@ class InferAndParseJsonEntry extends SugarEntryExtension with JsonEntryExtension optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { - val (locationArg, _) = InMemoryLocationValueBuilder.build(mandatoryArgs) + val codeData = getStringValue(mandatoryArgs.head) val preferNulls = optionalArgs.collectFirst { case a if a._1 == "preferNulls" => a._2 }.forall(getBoolValue) - val inferenceDiagnostic: Either[Seq[ErrorCompilerMessage], InputFormatDescriptor] = - getJsonInferrerProperties(Seq(locationArg), optionalArgs) - .flatMap(programContext.infer) - .left - .map(error => Seq(InvalidSemantic(node, error))) + val inferenceDiagnostic: Either[Seq[ErrorCompilerMessage], InputFormatDescriptor] = getJsonInferrerProperties( + Seq(ValueArg(Rql2LocationValue(new InMemoryByteStreamLocation(codeData), ""), Rql2LocationType())), + optionalArgs + ) + .flatMap(programContext.infer) + .left + .map(error => Seq(InvalidSemantic(node, error))) for ( descriptor <- inferenceDiagnostic; TextInputStreamFormatDescriptor( @@ -388,11 +393,13 @@ class InferAndParseJsonEntry extends SugarEntryExtension with JsonEntryExtension optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Exp = { - - val (locationArg, codeData) = InMemoryLocationValueBuilder.build(mandatoryArgs) + val codeData = getStringValue(mandatoryArgs.head) val inputFormatDescriptor = for ( - inferrerProperties <- getJsonInferrerProperties(Seq(locationArg), optionalArgs); + inferrerProperties <- getJsonInferrerProperties( + Seq(ValueArg(Rql2LocationValue(new InMemoryByteStreamLocation(codeData), ""), Rql2LocationType())), + optionalArgs + ); inputFormatDescriptor <- programContext.infer(inferrerProperties) ) yield { inputFormatDescriptor @@ -570,15 +577,15 @@ trait JsonEntryExtensionHelper extends EntryExtensionHelper { mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)] ): Either[String, JsonInferrerProperties] = { - Right( + getByteStreamLocation(mandatoryArgs.head).right.map { location => JsonInferrerProperties( - getLocationValue(mandatoryArgs.head), + location, optionalArgs.collectFirst { case a if a._1 == "sampleSize" => a._2 }.map(getIntValue), optionalArgs .collectFirst { case a if a._1 == "encoding" => a._2 } .map(v => getEncodingValue(v).fold(err => return Left(err), v => v)) ) - ) + } } // validates the type as entered by the user. We have the possibility to flag the error on the specific diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackage.scala index f6e7c07d7..55e1bd13e 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackage.scala @@ -14,7 +14,6 @@ package raw.compiler.rql2.builtin import raw.client.api._ import raw.compiler.base.source.Type -import raw.compiler.common.source._ import raw.compiler.rql2._ import raw.compiler.rql2.api.{Arg, EntryExtension, ExpParam, PackageExtension, Param} import raw.compiler.rql2.source._ @@ -29,12 +28,13 @@ class LocationPackage extends PackageExtension { } -class LocationBuildEntry extends EntryExtension { +class LocationFromStringEntry extends EntryExtension { override def packageName: String = "Location" - override def entryName: String = "Build" + override def entryName: String = "FromString" + // FIXME (msb): Make this a user-visible node and take advantage to document the exact format of the URLs allowed. override def docs: EntryDoc = ??? override def nrMandatoryParams: Int = 1 @@ -44,30 +44,7 @@ class LocationBuildEntry extends EntryExtension { Right(ExpParam(Rql2StringType())) } - override def optionalParams: Option[Set[String]] = Some(Set.empty) - - override def getOptionalParam(prevMandatoryArgs: Seq[Arg], idn: String): Either[String, Param] = { - Right( - ExpParam( - OneOfType( - Rql2IntType(), - Rql2StringType(), - Rql2BinaryType(), - Rql2BoolType(), - Rql2IntervalType(), - Rql2ListType( - Rql2RecordType( - Vector( - Rql2AttrType("_1", Rql2StringType(Set(Rql2IsNullableTypeProperty()))), - Rql2AttrType("_2", Rql2StringType(Set(Rql2IsNullableTypeProperty()))) - ) - ) - ), - Rql2ListType(Rql2IntType()) - ) - ) - ) - } + override def optionalParams: Option[Set[String]] = None override def returnType( mandatoryArgs: Seq[Arg], diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackageBuilder.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackageBuilder.scala index c949cf544..1dec654be 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackageBuilder.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/LocationPackageBuilder.scala @@ -16,11 +16,12 @@ import raw.compiler.common.source._ import raw.compiler.rql2.source._ object LocationPackageBuilder { - object Build { - def apply(url: Exp, args: Vector[(String, Exp)]): Exp = { + + object FromString { + def apply(url: Exp): Exp = { FunApp( - Proj(PackageIdnExp("Location"), "Build"), - Vector(FunAppArg(url, None)) ++ args.map { case (idn, e) => FunAppArg(e, Some(idn)) } + Proj(PackageIdnExp("Location"), "FromString"), + Vector(FunAppArg(url, None)) ) } } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/MySQLPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/MySQLPackage.scala index 792f52919..1b2198940 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/MySQLPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/MySQLPackage.scala @@ -12,14 +12,20 @@ package raw.compiler.rql2.builtin -import raw.compiler.base.errors.ErrorCompilerMessage +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ import raw.compiler.rql2.api._ import raw.compiler.rql2.ProgramContext import raw.compiler.rql2.source._ import raw.client.api._ -import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor} +import raw.inferrer.api.{ + SqlQueryInferrerProperties, + SqlQueryInputFormatDescriptor, + SqlTableInferrerProperties, + SqlTableInputFormatDescriptor +} +import raw.sources.jdbc.mysql.{MySqlServerLocation, MySqlTableLocation} class MySQLPackage extends PackageExtension { @@ -31,7 +37,7 @@ class MySQLPackage extends PackageExtension { } -class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class MySQLInferAndReadEntry extends SugarEntryExtension { override def packageName: String = "MySQL" @@ -119,22 +125,51 @@ class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionH ) } + private def getTableInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlTableInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val table = getStringValue(mandatoryArgs(1)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(3306) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new MySqlTableLocation(host, port, db, username, password, table)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: MySqlJdbcLocation) => + new MySqlTableLocation(l.host, l.port, l.database, l.username, l.password, table)(programContext.settings) + case Some(_) => return Left("not a MySQL server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlTableInferrerProperties(location, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, MySqlVendor()); + inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor + SqlTableInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } } } -class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class MySQLReadEntry extends SugarEntryExtension { override def packageName: String = "MySQL" @@ -217,6 +252,22 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } @@ -228,6 +279,7 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Exp = { + val db = FunAppArg(mandatoryArgs.head.asInstanceOf[ExpArg].e, None) val table = FunAppArg(mandatoryArgs(1).asInstanceOf[ExpArg].e, None) val tipe = FunAppArg(TypeExp(mandatoryArgs(2).asInstanceOf[TypeArg].t), None) @@ -244,7 +296,7 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { } } -class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class MySQLInferAndQueryEntry extends SugarEntryExtension { override def packageName: String = "MySQL" @@ -314,15 +366,44 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension } } + private def getQueryInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlQueryInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val query = getStringValue(mandatoryArgs(1)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(3306) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new MySqlServerLocation(host, port, db, username, password)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: MySqlJdbcLocation) => + new MySqlServerLocation(l.host, l.port, l.database, l.username, l.password)(programContext.settings) + case Some(_) => return Left("not a MySQL server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlQueryInferrerProperties(location, query, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, MySqlVendor()); + inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor + SqlQueryInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } @@ -348,7 +429,7 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension } -class MySQLQueryEntry extends EntryExtension with SqlTableExtensionHelper { +class MySQLQueryEntry extends EntryExtension { override def packageName: String = "MySQL" @@ -431,6 +512,22 @@ class MySQLQueryEntry extends EntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/OraclePackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/OraclePackage.scala index ac47358f4..9722496bc 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/OraclePackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/OraclePackage.scala @@ -12,7 +12,7 @@ package raw.compiler.rql2.builtin -import raw.compiler.base.errors.ErrorCompilerMessage +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ import raw.compiler.rql2.api.{ @@ -32,7 +32,13 @@ import raw.compiler.rql2.api.{ import raw.compiler.rql2.source._ import raw.compiler.rql2.ProgramContext import raw.client.api._ -import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor} +import raw.inferrer.api.{ + SqlQueryInferrerProperties, + SqlQueryInputFormatDescriptor, + SqlTableInferrerProperties, + SqlTableInputFormatDescriptor +} +import raw.sources.jdbc.oracle.{OracleServerLocation, OracleTableLocation} class OraclePackage extends PackageExtension { @@ -44,7 +50,7 @@ class OraclePackage extends PackageExtension { } -class OracleInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class OracleInferAndReadEntry extends SugarEntryExtension { override def packageName: String = "Oracle" @@ -138,22 +144,54 @@ class OracleInferAndReadEntry extends SugarEntryExtension with SqlTableExtension ) } + private def getTableInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlTableInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val schema = getStringValue(mandatoryArgs(1)) + val table = getStringValue(mandatoryArgs(2)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(1521) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new OracleTableLocation(host, port, db, username, password, schema, table)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: OracleJdbcLocation) => + new OracleTableLocation(l.host, l.port, l.database, l.username, l.password, schema, table)( + programContext.settings + ) + case Some(_) => return Left("not an Oracle server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlTableInferrerProperties(location, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, OracleVendor()); + inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor + SqlTableInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } } } -class OracleReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class OracleReadEntry extends SugarEntryExtension { override def packageName: String = "Oracle" @@ -242,6 +280,22 @@ class OracleReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(3).t validateTableType(t) } @@ -273,7 +327,7 @@ class OracleReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { } } -class OracleInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class OracleInferAndQueryEntry extends SugarEntryExtension { override def packageName: String = "Oracle" @@ -343,15 +397,44 @@ class OracleInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensio } } + private def getQueryInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlQueryInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val query = getStringValue(mandatoryArgs(1)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(1521) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new OracleServerLocation(host, port, db, username, password)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: OracleJdbcLocation) => + new OracleServerLocation(l.host, l.port, l.database, l.username, l.password)(programContext.settings) + case Some(_) => return Left("not an Oracle server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlQueryInferrerProperties(location, query, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, OracleVendor()); + inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor + SqlQueryInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } @@ -377,7 +460,7 @@ class OracleInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensio } -class OracleQueryEntry extends EntryExtension with SqlTableExtensionHelper { +class OracleQueryEntry extends EntryExtension { override def packageName: String = "Oracle" @@ -463,6 +546,22 @@ class OracleQueryEntry extends EntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/PostgreSQLPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/PostgreSQLPackage.scala index 1f48bd921..b986a3f4b 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/PostgreSQLPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/PostgreSQLPackage.scala @@ -12,7 +12,7 @@ package raw.compiler.rql2.builtin -import raw.compiler.base.errors.ErrorCompilerMessage +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ import raw.compiler.rql2._ @@ -32,7 +32,13 @@ import raw.compiler.rql2.api.{ } import raw.compiler.rql2.source._ import raw.client.api._ -import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor} +import raw.inferrer.api.{ + SqlQueryInferrerProperties, + SqlQueryInputFormatDescriptor, + SqlTableInferrerProperties, + SqlTableInputFormatDescriptor +} +import raw.sources.jdbc.pgsql.{PostgresqlServerLocation, PostgresqlTableLocation} class PostgreSQLPackage extends PackageExtension { @@ -44,7 +50,7 @@ class PostgreSQLPackage extends PackageExtension { } -class PostgreSQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class PostgreSQLInferAndReadEntry extends SugarEntryExtension { override def packageName: String = "PostgreSQL" @@ -138,22 +144,54 @@ class PostgreSQLInferAndReadEntry extends SugarEntryExtension with SqlTableExten ) } + private def getTableInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlTableInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val schema = getStringValue(mandatoryArgs(1)) + val table = getStringValue(mandatoryArgs(2)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(5432) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new PostgresqlTableLocation(host, port, db, username, password, schema, table)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: PostgresJdbcLocation) => + new PostgresqlTableLocation(l.host, l.port, l.database, l.username, l.password, schema, table)( + programContext.settings + ) + case Some(_) => return Left("not a PostgreSQL server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlTableInferrerProperties(location, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, PgSqlVendor()); + inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor + SqlTableInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } } } -class PostgreSQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class PostgreSQLReadEntry extends SugarEntryExtension { override def packageName: String = "PostgreSQL" @@ -245,6 +283,22 @@ class PostgreSQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelp optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(3).t validateTableType(t) } @@ -274,7 +328,7 @@ class PostgreSQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelp } } -class PostgreSQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class PostgreSQLInferAndQueryEntry extends SugarEntryExtension { override def packageName: String = "PostgreSQL" @@ -345,15 +399,44 @@ class PostgreSQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExte } } + private def getQueryInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlQueryInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val query = getStringValue(mandatoryArgs(1)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(5432) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new PostgresqlServerLocation(host, port, db, username, password)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: PostgresJdbcLocation) => + new PostgresqlServerLocation(l.host, l.port, l.database, l.username, l.password)(programContext.settings) + case Some(_) => return Left("not an Oracle server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlQueryInferrerProperties(location, query, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, PgSqlVendor()); + inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor + SqlQueryInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } @@ -379,7 +462,7 @@ class PostgreSQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExte } } -class PostgreSQLQueryEntry extends EntryExtension with SqlTableExtensionHelper { +class PostgreSQLQueryEntry extends EntryExtension { override def packageName: String = "PostgreSQL" @@ -466,6 +549,22 @@ class PostgreSQLQueryEntry extends EntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/S3Package.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/S3Package.scala index 7812785dd..d5ebd11a0 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/S3Package.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/S3Package.scala @@ -48,7 +48,7 @@ class S3BuildEntry extends EntryExtension { info = Some( "If the S3 bucket is not registered in the credentials storage, then the region, accessKey and secretKey must be provided as arguments." ), - examples = List(ExampleDoc("""S3.Location("S3://my-bucket/folder/sub-folder/file")""")), + examples = List(ExampleDoc("""S3.Build("S3://my-bucket/folder/sub-folder/file")""")), ret = Some(ReturnDoc("The S3 location.", retType = Some(TypeDoc(List("location"))))) ) diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SQLServerPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SQLServerPackage.scala index 3d436563f..c0581d988 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SQLServerPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SQLServerPackage.scala @@ -12,7 +12,7 @@ package raw.compiler.rql2.builtin -import raw.compiler.base.errors.ErrorCompilerMessage +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ import raw.compiler.rql2._ @@ -32,7 +32,13 @@ import raw.compiler.rql2.api.{ } import raw.compiler.rql2.source._ import raw.client.api._ -import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor} +import raw.inferrer.api.{ + SqlQueryInferrerProperties, + SqlQueryInputFormatDescriptor, + SqlTableInferrerProperties, + SqlTableInputFormatDescriptor +} +import raw.sources.jdbc.sqlserver.{SqlServerServerLocation, SqlServerTableLocation} class SQLServerPackage extends PackageExtension { @@ -44,7 +50,7 @@ class SQLServerPackage extends PackageExtension { } -class SQLServerInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SQLServerInferAndReadEntry extends SugarEntryExtension { override def packageName: String = "SQLServer" @@ -139,22 +145,54 @@ class SQLServerInferAndReadEntry extends SugarEntryExtension with SqlTableExtens ) } + private def getTableInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlTableInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val schema = getStringValue(mandatoryArgs(1)) + val table = getStringValue(mandatoryArgs(2)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(1433) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new SqlServerTableLocation(host, port, db, username, password, schema, table)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: SqlServerJdbcLocation) => + new SqlServerTableLocation(l.host, l.port, l.database, l.username, l.password, schema, table)( + programContext.settings + ) + case Some(_) => return Left("not an Oracle server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlTableInferrerProperties(location, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, SqlServerVendor()); + inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor + SqlTableInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } } } -class SQLServerReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SQLServerReadEntry extends SugarEntryExtension { override def packageName: String = "SQLServer" @@ -246,6 +284,22 @@ class SQLServerReadEntry extends SugarEntryExtension with SqlTableExtensionHelpe optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(3).t validateTableType(t) } @@ -276,7 +330,7 @@ class SQLServerReadEntry extends SugarEntryExtension with SqlTableExtensionHelpe } } -class SQLServerInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SQLServerInferAndQueryEntry extends SugarEntryExtension { override def packageName: String = "SQLServer" @@ -347,15 +401,44 @@ class SQLServerInferAndQueryEntry extends SugarEntryExtension with SqlTableExten } } + private def getQueryInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlQueryInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val query = getStringValue(mandatoryArgs(1)) + val location = + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val host = getStringValue(optionalArgs.find(_._1 == "host").getOrElse(return Left("host is required"))._2) + val port = optionalArgs.find(_._1 == "port").map(v => getIntValue(v._2)).getOrElse(1433) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new SqlServerServerLocation(host, port, db, username, password)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: SqlServerJdbcLocation) => + new SqlServerServerLocation(l.host, l.port, l.database, l.username, l.password)(programContext.settings) + case Some(_) => return Left("not an Oracle server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlQueryInferrerProperties(location, query, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, SqlServerVendor()); + inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor + SqlQueryInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, false, false) } @@ -380,7 +463,7 @@ class SQLServerInferAndQueryEntry extends SugarEntryExtension with SqlTableExten } } -class SQLServerQueryEntry extends EntryExtension with SqlTableExtensionHelper { +class SQLServerQueryEntry extends EntryExtension { override def packageName: String = "SQLServer" @@ -467,6 +550,22 @@ class SQLServerQueryEntry extends EntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "host") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "host")) { + return Left(Seq(InvalidSemantic(node, "host is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SnowflakePackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SnowflakePackage.scala index d753633af..f0a276073 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SnowflakePackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SnowflakePackage.scala @@ -12,7 +12,7 @@ package raw.compiler.rql2.builtin -import raw.compiler.base.errors.ErrorCompilerMessage +import raw.compiler.base.errors.{ErrorCompilerMessage, InvalidSemantic} import raw.compiler.base.source.{AnythingType, BaseNode, Type} import raw.compiler.common.source._ import raw.compiler.rql2.source._ @@ -26,6 +26,7 @@ import raw.compiler.rql2.api.{ Param, Rql2ListValue, Rql2OptionValue, + Rql2RecordAttr, Rql2RecordValue, Rql2StringValue, SugarEntryExtension, @@ -35,7 +36,13 @@ import raw.compiler.rql2.api.{ ValueParam } import raw.client.api._ -import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor} +import raw.inferrer.api.{ + SqlQueryInferrerProperties, + SqlQueryInputFormatDescriptor, + SqlTableInferrerProperties, + SqlTableInputFormatDescriptor +} +import raw.sources.jdbc.snowflake.{SnowflakeServerLocation, SnowflakeTableLocation} class SnowflakePackage extends PackageExtension { @@ -47,7 +54,7 @@ class SnowflakePackage extends PackageExtension { } -class SnowflakeInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SnowflakeInferAndReadEntry extends SugarEntryExtension { override def packageName: String = "Snowflake" @@ -148,13 +155,11 @@ class SnowflakeInferAndReadEntry extends SugarEntryExtension with SqlTableExtens val optArgs = optionalArgs.map { case (idn, ValueArg(Rql2StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) case (idn, ValueArg(Rql2ListValue(v: Seq[Rql2RecordValue]), _)) => - // building a List of tuples val records = v.map { r => val fields = r.v.zipWithIndex.map { - case (Rql2OptionValue(Some(Rql2StringValue(v))), idx) => + case (Rql2RecordAttr(_, Rql2OptionValue(Some(Rql2StringValue(v)))), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Build(StringConst(v)) - case (Rql2OptionValue(None), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Empty(t) - + case (Rql2RecordAttr(_, Rql2OptionValue(None)), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Empty(t) }.toVector RecordPackageBuilder.Build(fields) } @@ -167,22 +172,66 @@ class SnowflakeInferAndReadEntry extends SugarEntryExtension with SqlTableExtens ) } + private def getTableInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlTableInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val schema = getStringValue(mandatoryArgs(1)) + val table = getStringValue(mandatoryArgs(2)) + val parameters = + optionalArgs.collectFirst { case a if a._1 == "options" => getListKVValue(a._2) }.getOrElse(Seq.empty) + val location = + if ( + optionalArgs.exists(_._1 == "accountID") || optionalArgs.exists(_._1 == "username") || optionalArgs.exists( + _._1 == "password" + ) + ) { + val accountID = + getStringValue(optionalArgs.find(_._1 == "accountID").getOrElse(return Left("accountID is required"))._2) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new SnowflakeTableLocation(db, username, password, accountID, parameters.toMap, schema, table)( + programContext.settings + ) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: SnowflakeJdbcLocation) => new SnowflakeTableLocation( + l.database, + l.username, + l.password, + l.accountIdentifier, + l.parameters, + schema, + table + )( + programContext.settings + ) + case Some(_) => return Left("not a Snowflake server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlTableInferrerProperties(location, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, SnowflakeVendor()); + inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor + SqlTableInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, makeNullable = false, makeTryable = false) } } } -class SnowflakeReadEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SnowflakeReadEntry extends SugarEntryExtension { override def packageName: String = "Snowflake" @@ -285,6 +334,22 @@ class SnowflakeReadEntry extends SugarEntryExtension with SqlTableExtensionHelpe optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that accountID/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "accountID") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "accountID")) { + return Left(Seq(InvalidSemantic(node, "accountID is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(3).t validateTableType(t) } @@ -320,7 +385,7 @@ class SnowflakeReadEntry extends SugarEntryExtension with SqlTableExtensionHelpe } -class SnowflakeInferAndQueryEntry extends SugarEntryExtension with SqlTableExtensionHelper { +class SnowflakeInferAndQueryEntry extends SugarEntryExtension { override def packageName: String = "Snowflake" @@ -402,15 +467,48 @@ class SnowflakeInferAndQueryEntry extends SugarEntryExtension with SqlTableExten } } + private def getQueryInferrerProperties( + mandatoryArgs: Seq[Arg], + optionalArgs: Seq[(String, Arg)] + )(implicit programContext: ProgramContext): Either[String, SqlQueryInferrerProperties] = { + val db = getStringValue(mandatoryArgs(0)) + val query = getStringValue(mandatoryArgs(1)) + val parameters = + optionalArgs.collectFirst { case a if a._1 == "options" => getListKVValue(a._2) }.getOrElse(Seq.empty) + val location = + if ( + optionalArgs.exists(_._1 == "accountID") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + val accountID = + getStringValue(optionalArgs.find(_._1 == "accountID").getOrElse(return Left("accountID is required"))._2) + val username = + getStringValue(optionalArgs.find(_._1 == "username").getOrElse(return Left("username is required"))._2) + val password = + getStringValue(optionalArgs.find(_._1 == "password").getOrElse(return Left("password is required"))._2) + new SnowflakeServerLocation(db, username, password, accountID, parameters.toMap)(programContext.settings) + } else { + programContext.programEnvironment.jdbcServers.get(db) match { + case Some(l: SnowflakeJdbcLocation) => + new SnowflakeServerLocation(l.database, l.username, l.password, l.accountIdentifier, l.parameters)( + programContext.settings + ) + case Some(_) => return Left("not a Snowflake server") + case None => return Left(s"unknown database credential: $db") + } + } + Right(SqlQueryInferrerProperties(location, query, None)) + } + override def returnType( mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[String, Type] = { for ( - inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, SnowflakeVendor()); + inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs); inputFormatDescriptor <- programContext.infer(inferrerProperties); - SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor + SqlQueryInputFormatDescriptor(tipe) = inputFormatDescriptor ) yield { inferTypeToRql2Type(tipe, makeNullable = false, makeTryable = false) } @@ -432,9 +530,9 @@ class SnowflakeInferAndQueryEntry extends SugarEntryExtension with SqlTableExten // building a List of tuples val records = v.map { r => val fields = r.v.zipWithIndex.map { - case (Rql2OptionValue(Some(Rql2StringValue(v))), idx) => + case (Rql2RecordAttr(_, Rql2OptionValue(Some(Rql2StringValue(v)))), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Build(StringConst(v)) - case (Rql2OptionValue(None), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Empty(t) + case (Rql2RecordAttr(_, Rql2OptionValue(None)), idx) => s"_${idx + 1}" -> NullablePackageBuilder.Empty(t) }.toVector RecordPackageBuilder.Build(fields) @@ -450,7 +548,7 @@ class SnowflakeInferAndQueryEntry extends SugarEntryExtension with SqlTableExten } -class SnowflakeQueryEntry extends EntryExtension with SqlTableExtensionHelper { +class SnowflakeQueryEntry extends EntryExtension { override def packageName: String = "Snowflake" @@ -548,6 +646,22 @@ class SnowflakeQueryEntry extends EntryExtension with SqlTableExtensionHelper { optionalArgs: Seq[(String, Arg)], varArgs: Seq[Arg] )(implicit programContext: ProgramContext): Either[Seq[ErrorCompilerMessage], Type] = { + // Check that host/port/username/password are all present if any of them is present. + if ( + optionalArgs.exists(_._1 == "accountID") || optionalArgs + .exists(_._1 == "username") || optionalArgs.exists(_._1 == "password") + ) { + if (!optionalArgs.exists(_._1 == "accountID")) { + return Left(Seq(InvalidSemantic(node, "accountID is required"))) + } + if (!optionalArgs.exists(_._1 == "username")) { + return Left(Seq(InvalidSemantic(node, "username is required"))) + } + if (!optionalArgs.exists(_._1 == "password")) { + return Left(Seq(InvalidSemantic(node, "password is required"))) + } + } + val t = mandatoryArgs(2).t validateTableType(t) } diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SqlTableExtensionHelper.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SqlTableExtensionHelper.scala deleted file mode 100644 index a7fc79438..000000000 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/SqlTableExtensionHelper.scala +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.rql2.builtin - -import raw.compiler.base.errors.UnsupportedType -import raw.compiler.base.source.Type -import raw.compiler.rql2.api.{Arg, EntryExtensionHelper, Rql2StringValue, ValueArg} -import raw.compiler.rql2.source._ -import raw.inferrer.api.{SqlQueryInferrerProperties, SqlTableInferrerProperties} -import raw.client.api._ - -import scala.collection.mutable - -sealed trait SqlVendor -final case class MySqlVendor() extends SqlVendor -final case class PgSqlVendor() extends SqlVendor -final case class SqliteVendor() extends SqlVendor -final case class OracleVendor() extends SqlVendor -final case class SqlServerVendor() extends SqlVendor -final case class SparkSqlVendor() extends SqlVendor -final case class TeradataVendor() extends SqlVendor -final case class SnowflakeVendor() extends SqlVendor - -trait SqlTableExtensionHelper extends EntryExtensionHelper { - - protected def validateTableType(t: Type): Either[Seq[UnsupportedType], Rql2IterableType] = t match { - case Rql2IterableType(Rql2RecordType(atts, _), _) => - val validated = atts.map { x => - x.tipe match { - case _: Rql2StringType => Right(x) - case _: Rql2BoolType => Right(x) - case _: Rql2NumberType => Right(x) - // intervals are not supported so we cannot match temporals here - case _: Rql2DateType => Right(x) - case _: Rql2TimeType => Right(x) - case _: Rql2TimestampType => Right(x) - case _: Rql2BinaryType => Right(x) - case _ => Left(Seq(UnsupportedType(x.tipe, x.tipe, None))) - } - } - val errors = validated.collect { case Left(error) => error } - if (errors.nonEmpty) Left(errors.flatten) - else Right(Rql2IterableType(Rql2RecordType(atts))) - case _ => Left(Seq(UnsupportedType(t, t, None))) - } - - protected def getTableInferrerProperties( - mandatoryArgs: Seq[Arg], - optionalArgs: Seq[(String, Arg)], - vendor: SqlVendor - ): Either[String, SqlTableInferrerProperties] = { - - val tablePath = mandatoryArgs.map { case ValueArg(Rql2StringValue(v), _) => v }.mkString("/") - val url = vendorToUrl(vendor) + ":" + tablePath - val locationDesc = getLocation(url, optionalArgs.toMap) - Right(SqlTableInferrerProperties(locationDesc, None)) - } - - protected def getQueryInferrerProperties( - mandatoryArgs: Seq[Arg], - optionalArgs: Seq[(String, Arg)], - vendor: SqlVendor - ): Either[String, SqlQueryInferrerProperties] = { - - val db = getStringValue(mandatoryArgs.head) - val query = getStringValue(mandatoryArgs(1)) - val url = vendorToUrl(vendor) + ":" + db - val locationDesc = getLocation(url, optionalArgs.toMap) - Right(SqlQueryInferrerProperties(locationDesc, query, None)) - } - - private def getLocation(url: String, optionalArgs: Map[String, Arg]): LocationDescription = { - val locationSettings = mutable.HashMap[LocationSettingKey, LocationSettingValue]() - - Seq( - "db-host" -> optionalArgs.get("host").map(getStringValue), - "db-port" -> optionalArgs.get("port").map(getIntValue), - "db-account-id" -> optionalArgs.get("accountID").map(getStringValue), - "db-username" -> optionalArgs.get("username").map(getStringValue), - "db-password" -> optionalArgs.get("password").map(getStringValue), - "db-options" -> optionalArgs.get("options").map(getListKVValue) - ).foreach { - case (name, Some(value: String)) => locationSettings += LocationSettingKey(name) -> LocationStringSetting(value) - case (name, Some(value: Int)) => locationSettings += LocationSettingKey(name) -> LocationIntSetting(value) - case (name, Some(value: Seq[(String, String)])) => - locationSettings += LocationSettingKey(name) -> LocationKVSetting(value) - case (_, None) => - } - - LocationDescription(url, locationSettings.toMap) - } - - private def vendorToUrl(vendor: SqlVendor): String = { - vendor match { - case MySqlVendor() => "mysql" - case PgSqlVendor() => "pgsql" - case SqliteVendor() => "sqlite" - case OracleVendor() => "oracle" - case SqlServerVendor() => "sqlserver" - case SparkSqlVendor() => "sparksql" - case TeradataVendor() => "teradata" - case SnowflakeVendor() => "snowflake" - } - } - -} diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/StringPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/StringPackage.scala index b673c65a3..0714cf325 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/StringPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/StringPackage.scala @@ -46,7 +46,7 @@ class StringFromEntry extends EntryExtension { * Documentation. */ override def docs: EntryDoc = EntryDoc( - "Builds a string from a number, bool or temporal.", + "Builds a string from a number, bool, temporal or location.", params = List( ParamDoc("value", TypeDoc(List("number", "bool", "temporal")), "The value to convert to string.") ), @@ -75,7 +75,8 @@ class StringFromEntry extends EntryExtension { Rql2DateType(), Rql2TimeType(), Rql2TimestampType(), - Rql2IntervalType() + Rql2IntervalType(), + Rql2LocationType() ) ) ) diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/XmlPackage.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/XmlPackage.scala index fd3df8932..1a240c6b0 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/XmlPackage.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/XmlPackage.scala @@ -177,15 +177,15 @@ trait XmlEntryExtensionHelper extends EntryExtensionHelper { mandatoryArgs: Seq[Arg], optionalArgs: Seq[(String, Arg)] ): Either[String, XmlInferrerProperties] = { - Right( + getByteStreamLocation(mandatoryArgs.head).right.map { location => XmlInferrerProperties( - getLocationValue(mandatoryArgs.head), + location, optionalArgs.collectFirst { case a if a._1 == "sampleSize" => a._2 }.map(getIntValue), optionalArgs .collectFirst { case a if a._1 == "encoding" => a._2 } .map(v => getEncodingValue(v).fold(err => return Left(err), v => v)) ) - ) + } } protected def validateAttributeType(t: Type): Either[Seq[UnsupportedType], Type] = t match { diff --git a/snapi-frontend/src/main/scala/raw/compiler/rql2/source/SourceTree.scala b/snapi-frontend/src/main/scala/raw/compiler/rql2/source/SourceTree.scala index 9f1f8a80a..5fb832feb 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/rql2/source/SourceTree.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/rql2/source/SourceTree.scala @@ -15,6 +15,7 @@ package raw.compiler.rql2.source import org.bitbucket.inkytonik.kiama.output._ import raw.compiler.base.source.Type import raw.compiler.common.source._ +import raw.sources.api.Location /////////////////////////////////////////////////////////////////////////// // RQL2 @@ -276,6 +277,7 @@ final case class BoolConst(value: Boolean) extends Const final case class StringConst(value: String) extends Const final case class TripleQuotedStringConst(value: String) extends Const + final case class BinaryConst(bytes: Array[Byte]) extends Const { override def equals(obj: Any): Boolean = { obj match { @@ -285,6 +287,8 @@ final case class BinaryConst(bytes: Array[Byte]) extends Const { } } +final case class LocationConst(bytes: Array[Byte], publicDescription: String) extends Const + /** * Number Constants */ diff --git a/snapi-frontend/src/main/scala/raw/compiler/utils/package.scala b/snapi-frontend/src/main/scala/raw/compiler/utils/package.scala index 6fe75c917..7def9a357 100644 --- a/snapi-frontend/src/main/scala/raw/compiler/utils/package.scala +++ b/snapi-frontend/src/main/scala/raw/compiler/utils/package.scala @@ -12,8 +12,6 @@ package raw.compiler -import raw.sources.api.Location - import java.nio.file.Path package object utils { @@ -37,7 +35,6 @@ package object utils { private def doLookup(arg: Any): String = { arg match { - case l: Location => l.rawUri case p: Path => "file:" + p.toAbsolutePath.toString.replace("\\", "\\\\") case s: String => s case _ => arg.toString diff --git a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerProperties.scala b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerProperties.scala index 11067c7fc..b8c607afa 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerProperties.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerProperties.scala @@ -12,26 +12,26 @@ package raw.inferrer.api -import raw.client.api.LocationDescription import raw.sources.api._ +import raw.sources.bytestream.api.ByteStreamLocation +import raw.sources.filesystem.api.FileSystemLocation +import raw.sources.jdbc.api.{JdbcServerLocation, JdbcTableLocation} sealed trait InferrerProperties { - def location: LocationDescription - def maybeSampleSize: Option[Int] } -final case class SqlTableInferrerProperties(location: LocationDescription, maybeSampleSize: Option[Int]) +final case class SqlTableInferrerProperties(location: JdbcTableLocation, maybeSampleSize: Option[Int]) extends InferrerProperties final case class SqlQueryInferrerProperties( - location: LocationDescription, + location: JdbcServerLocation, sql: String, maybeSampleSize: Option[Int] ) extends InferrerProperties final case class CsvInferrerProperties( - location: LocationDescription, + location: ByteStreamLocation, maybeSampleSize: Option[Int], maybeEncoding: Option[Encoding], maybeHasHeader: Option[Boolean], @@ -44,7 +44,7 @@ final case class CsvInferrerProperties( ) extends InferrerProperties final case class ManyCsvInferrerProperties( - location: LocationDescription, + location: FileSystemLocation, maybeSampleSize: Option[Int], maybeSampleFiles: Option[Int], maybeEncoding: Option[Encoding], @@ -58,51 +58,51 @@ final case class ManyCsvInferrerProperties( ) extends InferrerProperties final case class HjsonInferrerProperties( - location: LocationDescription, + location: ByteStreamLocation, maybeSampleSize: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class ManyHjsonInferrerProperties( - location: LocationDescription, + location: FileSystemLocation, maybeSampleSize: Option[Int], maybeSampleFiles: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class JsonInferrerProperties( - location: LocationDescription, + location: ByteStreamLocation, maybeSampleSize: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class ManyJsonInferrerProperties( - location: LocationDescription, + location: FileSystemLocation, maybeSampleSize: Option[Int], maybeSampleFiles: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class XmlInferrerProperties( - location: LocationDescription, + location: ByteStreamLocation, maybeSampleSize: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class ManyXmlInferrerProperties( - location: LocationDescription, + location: FileSystemLocation, maybeSampleSize: Option[Int], maybeSampleFiles: Option[Int], maybeEncoding: Option[Encoding] ) extends InferrerProperties final case class AutoInferrerProperties( - location: LocationDescription, + location: Location, maybeSampleSize: Option[Int] ) extends InferrerProperties final case class ManyAutoInferrerProperties( - location: LocationDescription, + location: FileSystemLocation, maybeSampleSize: Option[Int], maybeSampleFiles: Option[Int] ) extends InferrerProperties diff --git a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerService.scala b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerService.scala index 2f36c69b2..fe000b405 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerService.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerService.scala @@ -14,8 +14,7 @@ package raw.inferrer.api import com.google.common.base.Stopwatch import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} -import raw.sources.api.SourceContext -import raw.utils.{RawException, RawService, RawUtils} +import raw.utils.{RawException, RawService, RawSettings, RawUtils} import java.util.concurrent.{ExecutionException, Executors, TimeUnit, TimeoutException} @@ -30,12 +29,10 @@ object InferrerService { private val prettyPrinter = new SourceTypePrettyPrinter } -abstract class InferrerService(implicit sourceContext: SourceContext) extends RawService { +abstract class InferrerService(implicit settings: RawSettings) extends RawService { import InferrerService._ - private val settings = sourceContext.settings - private val inferrerTimeoutMillis = settings.getDuration(INFERRER_TIMEOUT).toMillis private val inferrerExpirySeconds = settings.getDuration(INFERRER_EXPIRY).toSeconds diff --git a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerServiceProvider.scala b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerServiceProvider.scala index 431c7d117..24d46f329 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerServiceProvider.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/api/InferrerServiceProvider.scala @@ -13,15 +13,14 @@ package raw.inferrer.api import raw.inferrer.local.LocalInferrerService - -import raw.sources.api.SourceContext +import raw.utils.RawSettings object InferrerServiceProvider { private val INFERRER_IMPL = "raw.inferrer.impl" - def apply()(implicit sourceContext: SourceContext): InferrerService = { - sourceContext.settings.getStringOpt(INFERRER_IMPL) match { + def apply()(implicit settings: RawSettings): InferrerService = { + settings.getStringOpt(INFERRER_IMPL) match { case Some("local") => new LocalInferrerService() case Some(impl) => throw new InferrerException(s"cannot find inferrer service: $impl") case None => new LocalInferrerService() diff --git a/snapi-frontend/src/main/scala/raw/inferrer/api/InputFormatDescriptors.scala b/snapi-frontend/src/main/scala/raw/inferrer/api/InputFormatDescriptors.scala index 7f876f256..3a5eaec43 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/api/InputFormatDescriptors.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/api/InputFormatDescriptors.scala @@ -18,19 +18,9 @@ sealed trait InputFormatDescriptor { def tipe: SourceType } -final case class SqlTableInputFormatDescriptor( - vendor: String, - dbName: String, - maybeSchema: Option[String], - table: String, - tipe: SourceType -) extends InputFormatDescriptor +final case class SqlTableInputFormatDescriptor(tipe: SourceType) extends InputFormatDescriptor -final case class SqlQueryInputFormatDescriptor( - vendor: String, - dbName: String, - tipe: SourceType -) extends InputFormatDescriptor +final case class SqlQueryInputFormatDescriptor(tipe: SourceType) extends InputFormatDescriptor sealed trait InputStreamFormatDescriptor extends InputFormatDescriptor diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/EncodingInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/EncodingInferrer.scala index 2b85aa5f7..a1310674b 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/EncodingInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/EncodingInferrer.scala @@ -19,6 +19,7 @@ import org.apache.commons.io.ByteOrderMark import org.apache.commons.io.input.BOMInputStream import raw.sources.api._ import raw.sources.bytestream.api.SeekableInputStream +import raw.utils.RawSettings private[inferrer] case class TextBuffer(reader: Reader, encoding: Encoding, confidence: Int) @@ -30,9 +31,7 @@ private[inferrer] trait EncodingInferrer extends StrictLogging { import EncodingInferrer._ - protected def sourceContext: SourceContext - - protected val settings = sourceContext.settings + protected def settings: RawSettings private val encodingDetectionReadSize = settings.getBytes(ENCODING_DETECTION_READ_SIZE) diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala index e2abb4317..074adc8d4 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/LocalInferrerService.scala @@ -14,7 +14,8 @@ package raw.inferrer.local import com.typesafe.scalalogging.StrictLogging import org.bitbucket.inkytonik.kiama.output.PrettyPrinter -import raw.utils.RawException +import raw.compiler.rql2.api.LocationDescription +import raw.utils.{RawException, RawSettings} import raw.inferrer.api._ import raw.inferrer.local.auto.{AutoInferrer, InferrerBufferedSeekableIS} import raw.inferrer.local.csv.{CsvInferrer, CsvMergeTypes} @@ -24,20 +25,18 @@ import raw.inferrer.local.json.JsonInferrer import raw.inferrer.local.text.TextInferrer import raw.inferrer.local.xml.{XmlInferrer, XmlMergeTypes} import raw.sources.api._ -import raw.sources.bytestream.api.{ByteStreamLocation, ByteStreamLocationProvider} -import raw.sources.filesystem.api.FileSystemLocationProvider -import raw.sources.jdbc.api.{JdbcLocationProvider, JdbcTableLocationProvider} +import raw.sources.bytestream.api.ByteStreamLocation +import raw.sources.filesystem.api.FileSystemLocation +import raw.sources.jdbc.api.JdbcTableLocation import scala.util.control.NonFatal -class LocalInferrerService(implicit sourceContext: SourceContext) +class LocalInferrerService(implicit settings: RawSettings) extends InferrerService with InferrerErrorHandler with PrettyPrinter with StrictLogging { - private val settings = sourceContext.settings - private val textInferrer = new TextInferrer private val csvInferrer = new CsvInferrer private val jsonInferrer = new JsonInferrer @@ -56,7 +55,7 @@ class LocalInferrerService(implicit sourceContext: SourceContext) private val defaultSampleFiles = settings.getInt("raw.inferrer.local.sample-files") // This buffered-IS is only valid for text formats - private val useBufferedSeekableIs = sourceContext.settings.getBoolean("raw.inferrer.local.use-buffered-seekable-is") + private val useBufferedSeekableIs = settings.getBoolean("raw.inferrer.local.use-buffered-seekable-is") private def textInputStream(loc: ByteStreamLocation) = { if (useBufferedSeekableIs) { @@ -70,16 +69,13 @@ class LocalInferrerService(implicit sourceContext: SourceContext) try { properties match { case tbl: SqlTableInferrerProperties => - val location = JdbcTableLocationProvider.build(tbl.location) - val tipe = jdbcInferrer.getTableType(location) - SqlTableInputFormatDescriptor(location.vendor, location.dbName, location.maybeSchema, location.table, tipe) + val tipe = jdbcInferrer.getTableType(tbl.location) + SqlTableInputFormatDescriptor(tipe) case query: SqlQueryInferrerProperties => - val location = JdbcLocationProvider.build(query.location) - val tipe = jdbcInferrer.getQueryType(location, query.sql) - SqlQueryInputFormatDescriptor(location.vendor, location.dbName, tipe) + val tipe = jdbcInferrer.getQueryType(query.location, query.sql) + SqlQueryInputFormatDescriptor(tipe) case csv: CsvInferrerProperties => - val location = ByteStreamLocationProvider.build(csv.location) - val is = textInputStream(location) + val is = textInputStream(csv.location) try { csvInferrer.infer( is, @@ -97,8 +93,7 @@ class LocalInferrerService(implicit sourceContext: SourceContext) is.close() } case csv: ManyCsvInferrerProperties => - val location = FileSystemLocationProvider.build(csv.location) - val files = location.ls() + val files = csv.location.ls() readMany( files, csv.maybeSampleFiles, @@ -123,16 +118,14 @@ class LocalInferrerService(implicit sourceContext: SourceContext) } ) case hjson: HjsonInferrerProperties => - val location = ByteStreamLocationProvider.build(hjson.location) - val is = textInputStream(location) + val is = textInputStream(hjson.location) try { hjsonInferrer.infer(is, hjson.maybeEncoding, hjson.maybeSampleSize) } finally { is.close() } case hjson: ManyHjsonInferrerProperties => - val location = FileSystemLocationProvider.build(hjson.location) - val files = location.ls() + val files = hjson.location.ls() readMany( files, hjson.maybeSampleFiles, @@ -146,16 +139,14 @@ class LocalInferrerService(implicit sourceContext: SourceContext) } ) case json: JsonInferrerProperties => - val location = ByteStreamLocationProvider.build(json.location) - val is = textInputStream(location) + val is = textInputStream(json.location) try { jsonInferrer.infer(is, json.maybeEncoding, json.maybeSampleSize) } finally { is.close() } case json: ManyJsonInferrerProperties => - val location = FileSystemLocationProvider.build(json.location) - val files = location.ls() + val files = json.location.ls() readMany( files, json.maybeSampleFiles, @@ -169,16 +160,14 @@ class LocalInferrerService(implicit sourceContext: SourceContext) } ) case xml: XmlInferrerProperties => - val location = ByteStreamLocationProvider.build(xml.location) - val is = textInputStream(location) + val is = textInputStream(xml.location) try { xmlInferrer.infer(is, xml.maybeEncoding, xml.maybeSampleSize) } finally { is.close() } case xml: ManyXmlInferrerProperties => - val location = FileSystemLocationProvider.build(xml.location) - val files = location.ls() + val files = xml.location.ls() readMany( files, xml.maybeSampleFiles, @@ -191,20 +180,15 @@ class LocalInferrerService(implicit sourceContext: SourceContext) } } ) - case auto: AutoInferrerProperties => - if (ByteStreamLocationProvider.isSupported(auto.location)) { - val location = ByteStreamLocationProvider.build(auto.location) - autoInferrer.infer(location, auto.maybeSampleSize) - } else if (JdbcTableLocationProvider.isSupported(auto.location.url)) { - val location = JdbcTableLocationProvider.build(auto.location) - val tipe = jdbcInferrer.getTableType(location) - SqlTableInputFormatDescriptor(location.vendor, location.dbName, location.maybeSchema, location.table, tipe) - } else { - throw new LocalInferrerException("unsupported location for auto inference") + case auto: AutoInferrerProperties => auto.location match { + case bs: ByteStreamLocation => autoInferrer.infer(bs, auto.maybeSampleSize) + case tbl: JdbcTableLocation => + val tipe = jdbcInferrer.getTableType(tbl) + SqlTableInputFormatDescriptor(tipe) + case _ => throw new LocalInferrerException("unsupported location for auto inference") } case auto: ManyAutoInferrerProperties => - val location = FileSystemLocationProvider.build(auto.location) - val files = location.ls() + val files = auto.location.ls() readMany(files, auto.maybeSampleFiles, file => autoInferrer.infer(file, auto.maybeSampleSize)) } } catch { @@ -213,7 +197,7 @@ class LocalInferrerService(implicit sourceContext: SourceContext) // Errors such as e.g. accessing a data source are seen as inferrer errors. logger.debug( s"""Inferrer failed gracefully. - |Location: ${properties.location.url}""".stripMargin, + |Inferrer properties: $properties""".stripMargin, ex ) throw new LocalInferrerException(ex.getMessage, ex) @@ -244,9 +228,17 @@ class LocalInferrerService(implicit sourceContext: SourceContext) try { doInference(loc) } catch { - case ex: RawException => - // Annotate actual failing file in message - throw new LocationException(s"failed inferring '${loc.rawUri}' with error '${ex.getMessage}'", ex) + case ex: RawException => loc match { + case fs: FileSystemLocation => + // Annotate actual failing file in message. + throw new LocalInferrerException( + s"failed inferring '${LocationDescription.locationToPublicUrl(fs)}' with error '${ex.getMessage}'", + ex + ) + case _ => + // Otherwise, just leave message as is. + throw ex + } } case _ => throw new LocationException("input stream location required") } diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/auto/AutoInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/auto/AutoInferrer.scala index 5823ef603..58e65cab0 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/auto/AutoInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/auto/AutoInferrer.scala @@ -21,9 +21,9 @@ import raw.inferrer.local.hjson.HjsonInferrer import raw.inferrer.local.json.JsonInferrer import raw.inferrer.local.text.TextInferrer import raw.inferrer.local.xml.XmlInferrer -import raw.sources.api._ import raw.sources.bytestream.api.ByteStreamLocation import raw.sources.filesystem.api.{DirectoryMetadata, FileSystemLocation} +import raw.utils.RawSettings object AutoInferrer { private val USE_BUFFERED_SEEKABLE_IS = "raw.inferrer.local.use-buffered-seekable-is" @@ -35,7 +35,7 @@ class AutoInferrer( jsonInferrer: JsonInferrer, hjsonInferrer: HjsonInferrer, xmlInferrer: XmlInferrer -)(implicit protected val sourceContext: SourceContext) +)(implicit protected val settings: RawSettings) extends InferrerErrorHandler with EncodingInferrer with StrictLogging { @@ -43,33 +43,19 @@ class AutoInferrer( import AutoInferrer._ def infer(location: ByteStreamLocation, maybeSampleSize: Option[Int]): InputStreamFormatDescriptor = { - val maybeFileExtension = { - val i = location.rawUri.lastIndexOf('.') - if (i != -1) { - val ext = location.rawUri.substring(i + 1) - // If it looks like an extension, pass it to the inferrer - if (ext.length >= 3 && ext.length <= 8) Some(ext) - else None - } else { - None - } - } - - maybeFileExtension match { - case _ => location match { - case fs: FileSystemLocation => - // If it is a file system, check if it is a directory, to attempt to detect Hadoop-like files. - fs.metadata() match { - case DirectoryMetadata(_) => - throw new LocalInferrerException("automatic inference failed: location is a directory!") - case _ => - // Not a directory. - inferTextFormats(location, maybeSampleSize) - } + location match { + case fs: FileSystemLocation => + // If it is a file system, check if it is a directory, to attempt to detect Hadoop-like files. + fs.metadata() match { + case DirectoryMetadata(_) => + throw new LocalInferrerException("automatic inference failed: location is a directory!") case _ => - // Not a file system. + // Not a directory. inferTextFormats(location, maybeSampleSize) } + case _ => + // Not a file system. + inferTextFormats(location, maybeSampleSize) } } @@ -85,7 +71,7 @@ class AutoInferrer( // and pass those to inferrers. val is = - if (sourceContext.settings.getBoolean(USE_BUFFERED_SEEKABLE_IS)) { + if (settings.getBoolean(USE_BUFFERED_SEEKABLE_IS)) { new InferrerBufferedSeekableIS(location.getSeekableInputStream) } else { location.getSeekableInputStream diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/csv/CsvInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/csv/CsvInferrer.scala index b476f51ec..190777614 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/csv/CsvInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/csv/CsvInferrer.scala @@ -18,8 +18,8 @@ import raw.inferrer.local._ import raw.inferrer.local.text.TextLineIterator import raw.inferrer.api._ import raw.sources.bytestream.api.SeekableInputStream -import raw.sources.api.{Encoding, SourceContext} -import raw.utils.RawException +import raw.sources.api.Encoding +import raw.utils.{RawException, RawSettings} import scala.util.control.NonFatal @@ -29,7 +29,7 @@ object CsvInferrer { private val CSV_QUOTED_WEIGHT = "raw.inferrer.local.csv.quoted-weight" } -class CsvInferrer(implicit protected val sourceContext: SourceContext) +class CsvInferrer(implicit protected val settings: RawSettings) extends EncodingInferrer with InferrerErrorHandler with StrictLogging { diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/hjson/HjsonInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/hjson/HjsonInferrer.scala index 2db2d79de..ec34cd76d 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/hjson/HjsonInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/hjson/HjsonInferrer.scala @@ -20,7 +20,7 @@ import raw.inferrer.local.json.JsonUtils import raw.inferrer.local.text.TextLineIterator import raw.sources.api._ import raw.sources.bytestream.api.SeekableInputStream -import raw.utils.RawException +import raw.utils.{RawException, RawSettings} import java.io.Reader import scala.util.control.NonFatal @@ -29,7 +29,7 @@ object HjsonInferrer { private val HJSON_SAMPLE_SIZE = "raw.inferrer.local.hjson.sample-size" } -class HjsonInferrer(implicit protected val sourceContext: SourceContext) +class HjsonInferrer(implicit protected val settings: RawSettings) extends InferrerErrorHandler with EncodingInferrer with JsonUtils { diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/jdbc/JdbcInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/jdbc/JdbcInferrer.scala index 1417c9adb..f2bc01013 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/jdbc/JdbcInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/jdbc/JdbcInferrer.scala @@ -16,7 +16,7 @@ import java.sql.ResultSetMetaData import com.typesafe.scalalogging.StrictLogging import raw.inferrer.api.{SourceAttrType, SourceCollectionType, SourceRecordType, SourceType} -import raw.sources.jdbc.api.{JdbcLocation, JdbcTableLocation} +import raw.sources.jdbc.api.{JdbcServerLocation, JdbcTableLocation} import scala.collection.mutable @@ -26,7 +26,7 @@ class JdbcInferrer extends JdbcTypeToSourceType with StrictLogging { tableMetadataToSourceType(location.getType()) } - def getQueryType(location: JdbcLocation, query: String): SourceType = { + def getQueryType(location: JdbcServerLocation, query: String): SourceType = { location.jdbcClient.wrapSQLException { val conn = location.getJdbcConnection() try { diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/json/JsonInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/json/JsonInferrer.scala index 037d5f4f8..2be2a2f3f 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/json/JsonInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/json/JsonInferrer.scala @@ -19,7 +19,7 @@ import raw.inferrer.api._ import raw.inferrer.local._ import raw.sources.api._ import raw.sources.bytestream.api.SeekableInputStream -import raw.utils.RawException +import raw.utils.{RawException, RawSettings} import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal @@ -28,7 +28,7 @@ object JsonInferrer { private val JSON_SAMPLE_SIZE = "raw.inferrer.local.json.sample-size" } -class JsonInferrer(implicit protected val sourceContext: SourceContext) +class JsonInferrer(implicit protected val settings: RawSettings) extends InferrerErrorHandler with EncodingInferrer with TextTypeInferrer diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/text/TextInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/text/TextInferrer.scala index 2eea1f777..3a5e23a1e 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/text/TextInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/text/TextInferrer.scala @@ -18,7 +18,7 @@ import raw.inferrer.api._ import raw.inferrer.local._ import raw.sources.api._ import raw.sources.bytestream.api.SeekableInputStream -import raw.utils.RawException +import raw.utils.{RawException, RawSettings} import scala.util.control.NonFatal import scala.util.matching.Regex @@ -29,7 +29,7 @@ object TextInferrer { private val TEXT_SAMPLE_SIZE = "raw.inferrer.local.text.sample-size" } -class TextInferrer(implicit protected val sourceContext: SourceContext) +class TextInferrer(implicit protected val settings: RawSettings) extends InferrerErrorHandler with EncodingInferrer with StrictLogging { diff --git a/snapi-frontend/src/main/scala/raw/inferrer/local/xml/XmlInferrer.scala b/snapi-frontend/src/main/scala/raw/inferrer/local/xml/XmlInferrer.scala index da5126a11..92afbfdc6 100644 --- a/snapi-frontend/src/main/scala/raw/inferrer/local/xml/XmlInferrer.scala +++ b/snapi-frontend/src/main/scala/raw/inferrer/local/xml/XmlInferrer.scala @@ -17,7 +17,7 @@ import raw.inferrer.api._ import raw.inferrer.local._ import raw.sources.api._ import raw.sources.bytestream.api.SeekableInputStream -import raw.utils.RawException +import raw.utils.{RawException, RawSettings} import java.io.Reader import javax.xml.stream.XMLStreamException @@ -27,7 +27,7 @@ object XmlInferrer { private val XML_SAMPLE_SIZE = "raw.inferrer.local.xml.sample-size" } -class XmlInferrer(implicit protected val sourceContext: SourceContext) +class XmlInferrer(implicit protected val settings: RawSettings) extends InferrerErrorHandler with XmlMergeTypes with EncodingInferrer diff --git a/snapi-frontend/src/test/scala/raw/inferrer/local/LocalInferrerTest.scala b/snapi-frontend/src/test/scala/raw/inferrer/local/LocalInferrerTest.scala index 2eb89e5ff..f94e17ca2 100644 --- a/snapi-frontend/src/test/scala/raw/inferrer/local/LocalInferrerTest.scala +++ b/snapi-frontend/src/test/scala/raw/inferrer/local/LocalInferrerTest.scala @@ -13,7 +13,6 @@ package raw.inferrer.local import com.typesafe.scalalogging.StrictLogging -import raw.client.api._ import raw.utils._ import raw.inferrer.api._ import raw.sources.api._ @@ -42,11 +41,10 @@ class LocalInferrerTest extends RawTestSuite with SettingsTestContext with Stric out.write(s) out.close() val l1 = new LocalPath(f.toPath) - implicit val sourceContext = new SourceContext(null, null, settings) val inferrer = new LocalInferrerService try { val TextInputStreamFormatDescriptor(detectedEncoding, _, LinesInputFormatDescriptor(_, _, _)) = - inferrer.infer(AutoInferrerProperties(LocationDescription(l1.rawUri), None)) + inferrer.infer(AutoInferrerProperties(l1, None)) assert(detectedEncoding == encoding) } finally { RawUtils.withSuppressNonFatalException(inferrer.stop()) @@ -66,14 +64,13 @@ class LocalInferrerTest extends RawTestSuite with SettingsTestContext with Stric ex.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy) - implicit val sourceContext = new SourceContext(null, null, settings) val inferrer = new LocalInferrerService try { for (i <- 0 to 100) { ex.submit(new Runnable { override def run(): Unit = { val file = files(Random.nextInt(3)) - inferrer.infer(AutoInferrerProperties(LocationDescription(file.rawUri), None)) + inferrer.infer(AutoInferrerProperties(file, None)) } }) } @@ -159,7 +156,6 @@ class LocalInferrerTest extends RawTestSuite with SettingsTestContext with Stric ), false ) - implicit val sourceContext = new SourceContext(null, null, settings) val inferrer = new LocalInferrerService try { val result = inferrer.prettyPrint(tipe) diff --git a/snapi-frontend/src/test/scala/raw/inferrer/local/RD10260.scala b/snapi-frontend/src/test/scala/raw/inferrer/local/RD10260.scala index 92ccc1377..ff3b914fd 100644 --- a/snapi-frontend/src/test/scala/raw/inferrer/local/RD10260.scala +++ b/snapi-frontend/src/test/scala/raw/inferrer/local/RD10260.scala @@ -15,7 +15,6 @@ package raw.inferrer.local import com.typesafe.scalalogging.StrictLogging import raw.inferrer.api._ import raw.inferrer.local.json.JsonInferrer -import raw.sources.api.SourceContext import raw.utils.{RawTestSuite, SettingsTestContext} import java.io.StringReader @@ -28,7 +27,6 @@ class RD10260 extends RawTestSuite with SettingsTestContext with StrictLogging { | {"name": "Pont Neuf", "info": "48°51′24″N, 2°20′27″E"}, | {"name": "Eiffel Tower", "info": {"height": 300, "year": 1889}} |]""".stripMargin - implicit val sourceContext: SourceContext = new SourceContext(null, null, settings) val inferrer = new JsonInferrer val reader = new StringReader(json) val JsonInputFormatDescriptor(tipe, _, _, _, _) = inferrer.infer(reader, None) diff --git a/snapi-frontend/src/test/scala/raw/inferrer/local/RD10439.scala b/snapi-frontend/src/test/scala/raw/inferrer/local/RD10439.scala index 8f664951f..b7126ff96 100644 --- a/snapi-frontend/src/test/scala/raw/inferrer/local/RD10439.scala +++ b/snapi-frontend/src/test/scala/raw/inferrer/local/RD10439.scala @@ -13,10 +13,9 @@ package raw.inferrer.local import com.typesafe.scalalogging.StrictLogging -import raw.creds.api.MySqlCredential import raw.inferrer.api._ import raw.inferrer.local.jdbc.JdbcInferrer -import raw.sources.jdbc.mysql.{MySqlClient, MySqlTable} +import raw.sources.jdbc.mysql.{MySqlClient, MySqlTableLocation} import raw.utils.{RawTestSuite, SettingsTestContext} class RD10439 extends RawTestSuite with SettingsTestContext with StrictLogging { @@ -27,10 +26,9 @@ class RD10439 extends RawTestSuite with SettingsTestContext with StrictLogging { val mysqlPassword: String = sys.env("RAW_MYSQL_TEST_PASSWORD") test("infer mysql table which is repeated in another database") { _ => - val mysqlCreds = MySqlCredential(mysqlHostname, None, mysqlDb, Some(mysqlUsername), Some(mysqlPassword)) val inferrer = new JdbcInferrer() - val client = new MySqlClient(mysqlCreds) - val location = new MySqlTable(client, mysqlDb, "rd10439") + val client = new MySqlClient(mysqlHostname, 3306, mysqlDb, mysqlUsername, mysqlPassword) + val location = new MySqlTableLocation(client, "rd10439") val tipe = inferrer.getTableType(location) logger.info(s"tipe: $tipe") val expected = SourceCollectionType( diff --git a/snapi-frontend/src/test/scala/raw/inferrer/local/RD3852.scala b/snapi-frontend/src/test/scala/raw/inferrer/local/RD3852.scala index f2aa9b287..11e3128a5 100644 --- a/snapi-frontend/src/test/scala/raw/inferrer/local/RD3852.scala +++ b/snapi-frontend/src/test/scala/raw/inferrer/local/RD3852.scala @@ -13,9 +13,7 @@ package raw.inferrer.local import com.typesafe.scalalogging.StrictLogging -import raw.client.api.LocationDescription import raw.inferrer.api.{AutoInferrerProperties, CsvInputFormatDescriptor, TextInputStreamFormatDescriptor} -import raw.sources.api.SourceContext import raw.sources.filesystem.local.LocalLocationsTestContext import raw.utils.{RawTestSuite, RawUtils, SettingsTestContext} import raw.sources.filesystem.local.LocalPath @@ -26,14 +24,12 @@ class RD3852 extends RawTestSuite with SettingsTestContext with StrictLogging wi // So inferring this a csv file means that the other inferrers threw correctly // a LocalInferrerException while trying to parse the file test("Auto inferring CSV") { _ => - implicit val sourceContext = new SourceContext(null, null, settings) val inferrer = new LocalInferrerService val p = RawUtils.getResource("data/students/students.csv") val l1 = new LocalPath(p) try { - val TextInputStreamFormatDescriptor(_, _, format) = - inferrer.infer(AutoInferrerProperties(LocationDescription(l1.rawUri), None)) + val TextInputStreamFormatDescriptor(_, _, format) = inferrer.infer(AutoInferrerProperties(l1, None)) assert(format.isInstanceOf[CsvInputFormatDescriptor]) } finally { diff --git a/snapi-frontend/src/test/scala/raw/inferrer/local/json/JsonOrTypeTest.scala b/snapi-frontend/src/test/scala/raw/inferrer/local/json/JsonOrTypeTest.scala index 03c2ddd62..59e78fdec 100644 --- a/snapi-frontend/src/test/scala/raw/inferrer/local/json/JsonOrTypeTest.scala +++ b/snapi-frontend/src/test/scala/raw/inferrer/local/json/JsonOrTypeTest.scala @@ -14,31 +14,18 @@ package raw.inferrer.local.json import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import org.scalatest.matchers.{MatchResult, Matcher} -import raw.creds.api.CredentialsTestContext import raw.inferrer.api._ import raw.inferrer.local.LocalInferrerTestContext -import raw.sources.api.SourceContext import raw.utils._ import java.io.StringReader -class JsonOrTypeTest - extends RawTestSuite - with SettingsTestContext - with LocalInferrerTestContext - with CredentialsTestContext { - - implicit private var sourceContext: SourceContext = _ +class JsonOrTypeTest extends RawTestSuite with SettingsTestContext with LocalInferrerTestContext { private var inferrer: JsonInferrer = _ override def beforeAll(): Unit = { super.beforeAll() - sourceContext = new SourceContext( - InteractiveUser(Uid("janeUid"), "Jane Smith", "jane@example.com"), - credentials, - settings - ) inferrer = new JsonInferrer() } diff --git a/sources/src/test/scala/raw/sources/filesystem/dropbox/DropboxTestContext.scala b/snapi-frontend/src/test/scala/raw/sources/filesystem/dropbox/DropboxTestContext.scala similarity index 100% rename from sources/src/test/scala/raw/sources/filesystem/dropbox/DropboxTestContext.scala rename to snapi-frontend/src/test/scala/raw/sources/filesystem/dropbox/DropboxTestContext.scala diff --git a/sources/src/test/scala/raw/sources/filesystem/local/LocalLocationsTestContext.scala b/snapi-frontend/src/test/scala/raw/sources/filesystem/local/LocalLocationsTestContext.scala similarity index 100% rename from sources/src/test/scala/raw/sources/filesystem/local/LocalLocationsTestContext.scala rename to snapi-frontend/src/test/scala/raw/sources/filesystem/local/LocalLocationsTestContext.scala diff --git a/snapi-truffle/src/main/java/module-info.java b/snapi-truffle/src/main/java/module-info.java index 4c892eab3..46ea59b40 100644 --- a/snapi-truffle/src/main/java/module-info.java +++ b/snapi-truffle/src/main/java/module-info.java @@ -10,6 +10,8 @@ * licenses/APL.txt. */ +import raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationFromStringEntry; + module raw.snapi.truffle { // Direct dependencies requires java.base; @@ -48,15 +50,11 @@ requires ch.qos.logback.classic; requires com.google.common; requires jul.to.slf4j; + requires snowflake.jdbc; uses raw.compiler.rql2.api.EntryExtension; - uses raw.sources.jdbc.api.JdbcTableLocationBuilder; uses raw.compiler.rql2.api.PackageExtension; uses raw.client.api.CompilerServiceBuilder; - uses raw.sources.filesystem.api.FileSystemLocationBuilder; - uses raw.sources.bytestream.api.ByteStreamLocationBuilder; - uses raw.sources.jdbc.api.JdbcLocationBuilder; - uses raw.sources.jdbc.api.JdbcSchemaLocationBuilder; provides com.oracle.truffle.api.provider.TruffleLanguageProvider with raw.runtime.truffle.RawLanguageProvider; @@ -119,7 +117,7 @@ raw.compiler.snapi.truffle.builtin.list_extension.TruffleUnsafeFromListEntry, raw.compiler.snapi.truffle.builtin.list_extension.TruffleGroupListEntry, raw.compiler.snapi.truffle.builtin.list_extension.TruffleExistsListEntry, - raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationBuildEntry, + TruffleLocationFromStringEntry, raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationDescribeEntry, raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLsEntry, raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLlEntry, diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/http_extension/TruffleHttpCallEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/http_extension/TruffleHttpCallEntry.java index c4c50f2ef..b640b7c49 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/http_extension/TruffleHttpCallEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/http_extension/TruffleHttpCallEntry.java @@ -13,21 +13,17 @@ package raw.compiler.snapi.truffle.builtin.http_extension; import java.util.List; -import java.util.stream.Stream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.HttpCallEntry; -import raw.compiler.rql2.source.Rql2StringType; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; -import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; -import scala.collection.immutable.HashSet; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromHttpNode; -public abstract class TruffleHttpCallEntry extends HttpCallEntry implements TruffleEntryExtension { +public abstract class TruffleHttpCallEntry extends HttpCallEntry + implements TruffleEntryExtension, WithArgs { private final String method; @@ -36,56 +32,29 @@ public TruffleHttpCallEntry(String method) { this.method = method; } - private String replaceKey(String idn) { - return switch (idn) { - case "method" -> "http-method"; - case "bodyString" -> "http-body-string"; - case "bodyBinary" -> "http-body"; - case "token" -> "http-token"; - case "authCredentialName" -> "http-auth-cred-name"; - case "clientId" -> "http-client-id"; - case "clientSecret" -> "http-client-secret"; - case "authProvider" -> "http-auth-provider"; - case "tokenUrl" -> "http-token-url"; - case "useBasicAuth" -> "http-use-basic-auth"; - case "username" -> "http-user-name"; - case "password" -> "http-password"; - case "args" -> "http-args"; - case "headers" -> "http-headers"; - case "expectedStatus" -> "http-expected-status"; - default -> throw new RawTruffleInternalErrorException(); - }; - } - @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode url = args.get(0).exprNode(); - String[] keys = - Stream.concat( - args.stream() - .skip(1) - .filter(e -> e.identifier() != null) - .map(e -> replaceKey(e.identifier())), - Stream.of("http-method")) - .toArray(String[]::new); - - ExpressionNode[] values = - Stream.concat( - args.stream().skip(1).map(TruffleArg::exprNode), - Stream.of(new StringNode(this.method))) - .toArray(ExpressionNode[]::new); - - Rql2TypeWithProperties[] types = - Stream.concat( - args.stream() - .skip(1) - .filter(e -> e.identifier() != null && e.exprNode() != null) - .map(e -> (Rql2TypeWithProperties) e.type()), - Stream.of( - (Rql2TypeWithProperties) Rql2StringType.apply(new HashSet<>()))) - .toArray(Rql2TypeWithProperties[]::new); - - return new LocationBuildNode(url, keys, values, types); + ExpressionNode bodyString = arg(args, "bodyString").orElse(null); + ExpressionNode bodyBinary = arg(args, "bodyBinary").orElse(null); + ExpressionNode authCredentialName = arg(args, "authCredentialName").orElse(null); + ExpressionNode username = arg(args, "username").orElse(null); + ExpressionNode password = arg(args, "password").orElse(null); + ExpressionNode httpArgs = arg(args, "args").orElse(null); + ExpressionNode headers = arg(args, "headers").orElse(null); + ExpressionNode expectedStatus = arg(args, "expectedStatus").orElse(null); + + return new LocationFromHttpNode( + method, + url, + bodyString, + bodyBinary, + authCredentialName, + username, + password, + httpArgs, + headers, + expectedStatus); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/Jdbc.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/Jdbc.java index aff7a1ff2..4a4222ba3 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/Jdbc.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/Jdbc.java @@ -20,6 +20,7 @@ import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; import raw.runtime.truffle.ast.ProgramExpressionNode; +import raw.runtime.truffle.ast.expressions.builtin.jdbc.JdbcQueryNode; import raw.runtime.truffle.ast.io.csv.writer.internal.*; import raw.runtime.truffle.ast.io.jdbc.*; import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/WithJdbcArgs.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/WithJdbcArgs.java deleted file mode 100644 index d1268735b..000000000 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/jdbc/WithJdbcArgs.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.compiler.snapi.truffle.builtin.jdbc; - -import java.util.List; -import raw.compiler.snapi.truffle.TruffleArg; -import raw.compiler.snapi.truffle.builtin.WithArgs; -import raw.runtime.truffle.ExpressionNode; - -public interface WithJdbcArgs extends WithArgs { - default ExpressionNode host(List args) { - return arg(args, "host").orElse(null); - } - - default ExpressionNode port(List args) { - return arg(args, "port").orElse(null); - } - - default ExpressionNode username(List args) { - return arg(args, "username").orElse(null); - } - - default ExpressionNode password(List args) { - return arg(args, "password").orElse(null); - } - - default ExpressionNode accountID(List args) { - return arg(args, "accountID").orElse(null); - } - - default ExpressionNode options(List args) { - return arg(args, "options").orElse(null); - } -} diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationBuildEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationFromStringEntry.java similarity index 62% rename from snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationBuildEntry.java rename to snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationFromStringEntry.java index 572356840..57ebd9d16 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationBuildEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/location_extension/TruffleLocationFromStringEntry.java @@ -14,29 +14,19 @@ import java.util.List; import raw.compiler.base.source.Type; -import raw.compiler.rql2.builtin.LocationBuildEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; +import raw.compiler.rql2.builtin.LocationFromStringEntry; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromStringNode; -public class TruffleLocationBuildEntry extends LocationBuildEntry +public class TruffleLocationFromStringEntry extends LocationFromStringEntry implements TruffleEntryExtension, WithArgs { @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { - String[] keys = - args.stream() - .filter(a -> a.identifier() != null) - .map(a -> a.identifier().replace('_', '-')) - .toArray(String[]::new); - - ExpressionNode[] values = optionalArgs(args); - - Rql2TypeWithProperties[] types = optionalArgsTypes(args); - - return new LocationBuildNode(args.get(0).exprNode(), keys, values, types); + ExpressionNode url = args.get(0).exprNode(); + return new LocationFromStringNode(url); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/mysql_extension/TruffleMySQLQueryEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/mysql_extension/TruffleMySQLQueryEntry.java index 1bba10d54..70574a0cb 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/mysql_extension/TruffleMySQLQueryEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/mysql_extension/TruffleMySQLQueryEntry.java @@ -13,54 +13,39 @@ package raw.compiler.snapi.truffle.builtin.mysql_extension; import java.util.List; -import java.util.stream.IntStream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.MySQLQueryEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.compiler.snapi.truffle.builtin.jdbc.Jdbc; -import raw.compiler.snapi.truffle.builtin.jdbc.WithJdbcArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.binary.PlusNode; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromMySQLCredentialNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromMySQLNode; +import raw.runtime.truffle.ast.expressions.literals.IntNode; import raw.runtime.truffle.runtime.exceptions.rdbms.MySQLExceptionHandler; public class TruffleMySQLQueryEntry extends MySQLQueryEntry - implements TruffleEntryExtension, WithJdbcArgs { + implements TruffleEntryExtension, WithArgs { + @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode db = args.get(0).exprNode(); + ExpressionNode query = args.get(1).exprNode(); - String[] allKeys = new String[] {"db-host", "db-port", "db-username", "db-password"}; - ExpressionNode[] allValues = - new ExpressionNode[] {host(args), port(args), username(args), password(args)}; - - String[] keys = - IntStream.range(0, allKeys.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allKeys[i]) - .toArray(String[]::new); - - ExpressionNode[] values = - IntStream.range(0, allValues.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allValues[i]) - .toArray(ExpressionNode[]::new); - - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(a -> a.identifier() != null) - .map(a -> (Rql2TypeWithProperties) a.type()) - .toArray(Rql2TypeWithProperties[]::new); + ExpressionNode location; - LocationBuildNode location = - new LocationBuildNode(new PlusNode(new StringNode("mysql:"), db), keys, values, types); + ExpressionNode host = arg(args, "host").orElse(null); + if (host == null) { + location = new LocationFromMySQLCredentialNode(db); + } else { + ExpressionNode port = arg(args, "port").orElse(new IntNode(3306)); + ExpressionNode username = arg(args, "username").get(); + ExpressionNode password = arg(args, "password").get(); + location = new LocationFromMySQLNode(host, port, db, username, password); + } - return Jdbc.query( - location, args.get(1).exprNode(), type, new MySQLExceptionHandler(), rawLanguage); + return Jdbc.query(location, query, type, new MySQLExceptionHandler(), rawLanguage); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/oracle_extension/TruffleOracleQueryEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/oracle_extension/TruffleOracleQueryEntry.java index 0712e705b..78ac05277 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/oracle_extension/TruffleOracleQueryEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/oracle_extension/TruffleOracleQueryEntry.java @@ -13,54 +13,39 @@ package raw.compiler.snapi.truffle.builtin.oracle_extension; import java.util.List; -import java.util.stream.IntStream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.OracleQueryEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.compiler.snapi.truffle.builtin.jdbc.Jdbc; -import raw.compiler.snapi.truffle.builtin.jdbc.WithJdbcArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.binary.PlusNode; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; -import raw.runtime.truffle.runtime.exceptions.rdbms.MySQLExceptionHandler; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromOracleCredentialNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromOracleNode; +import raw.runtime.truffle.ast.expressions.literals.IntNode; +import raw.runtime.truffle.runtime.exceptions.rdbms.OracleExceptionHandler; public class TruffleOracleQueryEntry extends OracleQueryEntry - implements TruffleEntryExtension, WithJdbcArgs { + implements TruffleEntryExtension, WithArgs { + @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode db = args.get(0).exprNode(); + ExpressionNode query = args.get(1).exprNode(); - String[] allKeys = new String[] {"db-host", "db-port", "db-username", "db-password"}; - ExpressionNode[] allValues = - new ExpressionNode[] {host(args), port(args), username(args), password(args)}; - - String[] keys = - IntStream.range(0, allKeys.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allKeys[i]) - .toArray(String[]::new); - - ExpressionNode[] values = - IntStream.range(0, allValues.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allValues[i]) - .toArray(ExpressionNode[]::new); - - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(a -> a.identifier() != null) - .map(a -> (Rql2TypeWithProperties) a.type()) - .toArray(Rql2TypeWithProperties[]::new); + ExpressionNode location; - LocationBuildNode location = - new LocationBuildNode(new PlusNode(new StringNode("oracle:"), db), keys, values, types); + ExpressionNode host = arg(args, "host").orElse(null); + if (host == null) { + location = new LocationFromOracleCredentialNode(db); + } else { + ExpressionNode port = arg(args, "port").orElse(new IntNode(1521)); + ExpressionNode username = arg(args, "username").get(); + ExpressionNode password = arg(args, "password").get(); + location = new LocationFromOracleNode(host, port, db, username, password); + } - return Jdbc.query( - location, args.get(1).exprNode(), type, new MySQLExceptionHandler(), rawLanguage); + return Jdbc.query(location, query, type, new OracleExceptionHandler(), rawLanguage); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/postgresql_extension/TrufflePostgreSQLQueryEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/postgresql_extension/TrufflePostgreSQLQueryEntry.java index 54101d0d9..344cd843a 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/postgresql_extension/TrufflePostgreSQLQueryEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/postgresql_extension/TrufflePostgreSQLQueryEntry.java @@ -13,54 +13,39 @@ package raw.compiler.snapi.truffle.builtin.postgresql_extension; import java.util.List; -import java.util.stream.IntStream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.PostgreSQLQueryEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.compiler.snapi.truffle.builtin.jdbc.Jdbc; -import raw.compiler.snapi.truffle.builtin.jdbc.WithJdbcArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.binary.PlusNode; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; -import raw.runtime.truffle.runtime.exceptions.rdbms.MySQLExceptionHandler; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromPostgreSQLCredentialNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromPostgreSQLNode; +import raw.runtime.truffle.ast.expressions.literals.IntNode; +import raw.runtime.truffle.runtime.exceptions.rdbms.PostgreSQLExceptionHandler; public class TrufflePostgreSQLQueryEntry extends PostgreSQLQueryEntry - implements TruffleEntryExtension, WithJdbcArgs { + implements TruffleEntryExtension, WithArgs { + @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode db = args.get(0).exprNode(); + ExpressionNode query = args.get(1).exprNode(); - String[] allKeys = new String[] {"db-host", "db-port", "db-username", "db-password"}; - ExpressionNode[] allValues = - new ExpressionNode[] {host(args), port(args), username(args), password(args)}; - - String[] keys = - IntStream.range(0, allKeys.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allKeys[i]) - .toArray(String[]::new); - - ExpressionNode[] values = - IntStream.range(0, allValues.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allValues[i]) - .toArray(ExpressionNode[]::new); - - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(a -> a.identifier() != null) - .map(a -> (Rql2TypeWithProperties) a.type()) - .toArray(Rql2TypeWithProperties[]::new); + ExpressionNode location; - LocationBuildNode location = - new LocationBuildNode(new PlusNode(new StringNode("pgsql:"), db), keys, values, types); + ExpressionNode host = arg(args, "host").orElse(null); + if (host == null) { + location = new LocationFromPostgreSQLCredentialNode(db); + } else { + ExpressionNode port = arg(args, "port").orElse(new IntNode(5432)); + ExpressionNode username = arg(args, "username").get(); + ExpressionNode password = arg(args, "password").get(); + location = new LocationFromPostgreSQLNode(host, port, db, username, password); + } - return Jdbc.query( - location, args.get(1).exprNode(), type, new MySQLExceptionHandler(), rawLanguage); + return Jdbc.query(location, query, type, new PostgreSQLExceptionHandler(), rawLanguage); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/s3_extension/TruffleS3BuildEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/s3_extension/TruffleS3BuildEntry.java index 801b45d19..317e659d1 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/s3_extension/TruffleS3BuildEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/s3_extension/TruffleS3BuildEntry.java @@ -12,47 +12,25 @@ package raw.compiler.snapi.truffle.builtin.s3_extension; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.S3BuildEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromS3Node; -public class TruffleS3BuildEntry extends S3BuildEntry implements TruffleEntryExtension { - - private static final Map keyMap = - new HashMap<>() { - { - put("region", "s3-region"); - put("accessKey", "s3-access-key"); - put("secretKey", "s3-secret-key"); - } - }; +public class TruffleS3BuildEntry extends S3BuildEntry implements TruffleEntryExtension, WithArgs { public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode url = args.get(0).exprNode(); - String[] keys = - args.stream() - .skip(1) - .map(TruffleArg::identifier) - .filter(Objects::nonNull) - .map(keyMap::get) - .toArray(String[]::new); - ExpressionNode[] values = - args.stream().skip(1).map(TruffleArg::exprNode).toArray(ExpressionNode[]::new); - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(arg -> arg.identifier() != null) - .map(arg -> (Rql2TypeWithProperties) arg.type()) - .toArray(Rql2TypeWithProperties[]::new); - return new LocationBuildNode(url, keys, values, types); + + ExpressionNode accessKey = arg(args, "accessKey").orElse(null); + ExpressionNode secretKey = arg(args, "secretKey").orElse(null); + ExpressionNode region = arg(args, "region").orElse(null); + + return new LocationFromS3Node(url, accessKey, secretKey, region); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/snowflake_extension/TruffleSnowflakeQueryEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/snowflake_extension/TruffleSnowflakeQueryEntry.java index e1233881a..1148b98d5 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/snowflake_extension/TruffleSnowflakeQueryEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/snowflake_extension/TruffleSnowflakeQueryEntry.java @@ -13,58 +13,38 @@ package raw.compiler.snapi.truffle.builtin.snowflake_extension; import java.util.List; -import java.util.stream.IntStream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.SnowflakeQueryEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.compiler.snapi.truffle.builtin.jdbc.Jdbc; -import raw.compiler.snapi.truffle.builtin.jdbc.WithJdbcArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.binary.PlusNode; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; -import raw.runtime.truffle.runtime.exceptions.rdbms.MySQLExceptionHandler; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromSnowflakeCredentialNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromSnowflakeNode; +import raw.runtime.truffle.runtime.exceptions.rdbms.SnowflakeExceptionHandler; public class TruffleSnowflakeQueryEntry extends SnowflakeQueryEntry - implements TruffleEntryExtension, WithJdbcArgs { + implements TruffleEntryExtension, WithArgs { + @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode db = args.get(0).exprNode(); - String[] allKeys = - new String[] { - "db-host", "db-port", "db-username", "db-password", "db-account-id", "db-options" - }; - ExpressionNode[] allValues = - new ExpressionNode[] { - host(args), port(args), username(args), password(args), accountID(args), options(args) - }; - - String[] keys = - IntStream.range(0, allKeys.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allKeys[i]) - .toArray(String[]::new); - - ExpressionNode[] values = - IntStream.range(0, allValues.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allValues[i]) - .toArray(ExpressionNode[]::new); + ExpressionNode query = args.get(1).exprNode(); - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(a -> a.identifier() != null) - .map(a -> (Rql2TypeWithProperties) a.type()) - .toArray(Rql2TypeWithProperties[]::new); + ExpressionNode location; - LocationBuildNode location = - new LocationBuildNode(new PlusNode(new StringNode("snowflake:"), db), keys, values, types); + ExpressionNode accountID = arg(args, "accountID").orElse(null); + if (accountID == null) { + location = new LocationFromSnowflakeCredentialNode(db); + } else { + ExpressionNode username = arg(args, "username").get(); + ExpressionNode password = arg(args, "password").get(); + ExpressionNode options = arg(args, "options").orElse(null); + location = new LocationFromSnowflakeNode(db, username, password, accountID, options); + } - return Jdbc.query( - location, args.get(1).exprNode(), type, new MySQLExceptionHandler(), rawLanguage); + return Jdbc.query(location, query, type, new SnowflakeExceptionHandler(), rawLanguage); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/sqlserver_extension/TruffleSQLServerQueryEntry.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/sqlserver_extension/TruffleSQLServerQueryEntry.java index 509d4b324..92f870742 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/sqlserver_extension/TruffleSQLServerQueryEntry.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/builtin/sqlserver_extension/TruffleSQLServerQueryEntry.java @@ -13,53 +13,39 @@ package raw.compiler.snapi.truffle.builtin.sqlserver_extension; import java.util.List; -import java.util.stream.IntStream; import raw.compiler.base.source.Type; import raw.compiler.rql2.builtin.SQLServerQueryEntry; -import raw.compiler.rql2.source.Rql2TypeWithProperties; import raw.compiler.snapi.truffle.TruffleArg; import raw.compiler.snapi.truffle.TruffleEntryExtension; +import raw.compiler.snapi.truffle.builtin.WithArgs; import raw.compiler.snapi.truffle.builtin.jdbc.Jdbc; -import raw.compiler.snapi.truffle.builtin.jdbc.WithJdbcArgs; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; -import raw.runtime.truffle.ast.expressions.binary.PlusNode; -import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationBuildNode; -import raw.runtime.truffle.ast.expressions.literals.StringNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromSQLServerCredentialNode; +import raw.runtime.truffle.ast.expressions.builtin.location_package.LocationFromSQLServerNode; +import raw.runtime.truffle.ast.expressions.literals.IntNode; import raw.runtime.truffle.runtime.exceptions.rdbms.SqlServerExceptionHandler; public class TruffleSQLServerQueryEntry extends SQLServerQueryEntry - implements TruffleEntryExtension, WithJdbcArgs { + implements TruffleEntryExtension, WithArgs { + @Override public ExpressionNode toTruffle(Type type, List args, RawLanguage rawLanguage) { ExpressionNode db = args.get(0).exprNode(); + ExpressionNode query = args.get(1).exprNode(); - String[] allKeys = new String[] {"db-host", "db-port", "db-username", "db-password"}; - ExpressionNode[] allValues = - new ExpressionNode[] {host(args), port(args), username(args), password(args)}; + ExpressionNode location; - String[] keys = - IntStream.range(0, allKeys.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allKeys[i]) - .toArray(String[]::new); + ExpressionNode host = arg(args, "host").orElse(null); + if (host == null) { + location = new LocationFromSQLServerCredentialNode(db); + } else { + ExpressionNode port = arg(args, "port").orElse(new IntNode(1433)); + ExpressionNode username = arg(args, "username").get(); + ExpressionNode password = arg(args, "password").get(); + location = new LocationFromSQLServerNode(host, port, db, username, password); + } - ExpressionNode[] values = - IntStream.range(0, allValues.length) - .filter(i -> allValues[i] != null) - .mapToObj(i -> allValues[i]) - .toArray(ExpressionNode[]::new); - - Rql2TypeWithProperties[] types = - args.stream() - .skip(1) - .filter(a -> a.identifier() != null) - .map(a -> (Rql2TypeWithProperties) a.type()) - .toArray(Rql2TypeWithProperties[]::new); - - ExpressionNode location = - new LocationBuildNode(new PlusNode(new StringNode("sqlserver:"), db), keys, values, types); - return Jdbc.query( - location, args.get(1).exprNode(), type, new SqlServerExceptionHandler(), rawLanguage); + return Jdbc.query(location, query, type, new SqlServerExceptionHandler(), rawLanguage); } } diff --git a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/compiler/SnapiTruffleEmitter.java b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/compiler/SnapiTruffleEmitter.java index 19e900012..f0778b82d 100644 --- a/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/compiler/SnapiTruffleEmitter.java +++ b/snapi-truffle/src/main/java/raw/compiler/snapi/truffle/compiler/SnapiTruffleEmitter.java @@ -17,7 +17,6 @@ import com.oracle.truffle.api.RootCallTarget; import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.FrameSlotKind; -import com.oracle.truffle.api.nodes.Node; import org.bitbucket.inkytonik.kiama.relation.TreeRelation; import org.bitbucket.inkytonik.kiama.util.Entity; import raw.compiler.base.source.Type; @@ -30,7 +29,7 @@ import raw.compiler.rql2.source.*; import raw.compiler.snapi.truffle.TruffleEmitter; import raw.compiler.snapi.truffle.TruffleEntryExtension; -import raw.compiler.snapi.truffle.builtin.test_extension.TruffleVarNullableStringExpTestEntry; +import raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationFromStringEntry; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawLanguage; import raw.runtime.truffle.StatementNode; @@ -54,7 +53,6 @@ import raw.runtime.truffle.ast.local.WriteLocalVariableNodeGen; import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; import raw.runtime.truffle.runtime.function.Function; -import scala.Option; import scala.collection.JavaConverters; public class SnapiTruffleEmitter extends TruffleEmitter { @@ -131,7 +129,7 @@ public class SnapiTruffleEmitter extends TruffleEmitter { new raw.compiler.snapi.truffle.builtin.list_extension.TruffleUnsafeFromListEntry(), new raw.compiler.snapi.truffle.builtin.list_extension.TruffleGroupListEntry(), new raw.compiler.snapi.truffle.builtin.list_extension.TruffleExistsListEntry(), - new raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationBuildEntry(), + new TruffleLocationFromStringEntry(), new raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationDescribeEntry(), new raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLsEntry(), new raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLlEntry(), @@ -530,7 +528,6 @@ public ClosureNode recurseLambda(TruffleBuildBody truffleBuildBody) { return new ClosureNode(f, new ExpressionNode[]{null}); } - public ExpressionNode recurseExp(Exp in) { return switch (in) { case Exp ignored when tipe(in) instanceof PackageType || tipe(in) instanceof PackageEntryType -> @@ -564,6 +561,7 @@ case Exp ignored when tipe(in) instanceof PackageType || tipe(in) instanceof Pac default -> throw new RawTruffleInternalErrorException(); }; case BinaryConst bc -> new BinaryConstNode(bc.bytes()); + case LocationConst lc -> new LocationConstNode(lc.bytes(), lc.publicDescription()); case UnaryExp ue -> switch (ue.unaryOp()) { case Neg ignored -> NegNodeGen.create(recurseExp(ue.exp())); case Not ignored -> NotNodeGen.create(recurseExp(ue.exp())); diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/RawContext.java b/snapi-truffle/src/main/java/raw/runtime/truffle/RawContext.java index c3e80e8f1..d45b92a6d 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/RawContext.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/RawContext.java @@ -18,24 +18,15 @@ import com.oracle.truffle.api.interop.TruffleObject; import com.oracle.truffle.api.nodes.Node; import java.io.OutputStream; -import java.util.Objects; +import java.util.Map; +import java.util.Set; import raw.client.api.*; -import raw.creds.api.Secret; import raw.inferrer.api.InferrerService; import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; import raw.runtime.truffle.runtime.function.RawFunctionRegistry; -import raw.sources.api.SourceContext; -import raw.utils.AuthenticatedUser; -import raw.utils.InteractiveUser; import raw.utils.RawSettings; -import scala.Option; -import scala.Some; -import scala.Tuple2; +import raw.utils.RawUid; import scala.collection.JavaConverters; -import scala.collection.immutable.Map; -import scala.collection.immutable.Seq; -import scala.collection.immutable.Seq$; -import scala.collection.immutable.Set; public final class RawContext { @@ -43,9 +34,6 @@ public final class RawContext { private final Env env; private final RawSettings rawSettings; private final OutputStream output; - private final AuthenticatedUser user; - private final String traceId; - private final String[] scopes; private final ProgramEnvironment programEnvironment; private final RawFunctionRegistry functionRegistry; @@ -66,39 +54,10 @@ public RawContext(RawLanguage language, Env env) { this.rawSettings = new RawSettings(rawSettingsConfigString); } - // Set user from environment variable. - String uid = Objects.toString(env.getEnvironment().get("RAW_USER"), ""); - this.user = new InteractiveUser(uid, uid, uid, (Seq) Seq$.MODULE$.empty()); - - // Set traceId from environment variable. - String traceId = Objects.toString(env.getEnvironment().get("RAW_TRACE_ID"), ""); - this.traceId = traceId; - - // Set scopes from environment variable. - String scopesStr = Objects.toString(env.getEnvironment().get("RAW_SCOPES"), ""); - this.scopes = (scopesStr == null || scopesStr.isEmpty()) ? new String[0] : scopesStr.split(","); - - // Create program environment. - Set scalaScopes = - JavaConverters.asScalaSetConverter(java.util.Set.of(this.scopes)).asScala().toSet(); - - java.util.Map javaOptions = new java.util.HashMap<>(); - env.getOptions() - .getDescriptors() - .forEach(d -> javaOptions.put(d.getName(), env.getOptions().get(d.getKey()).toString())); - - Map scalaOptions = - JavaConverters.mapAsScalaMapConverter(javaOptions) - .asScala() - .toMap(scala.Predef.>conforms()); - - Option maybeTraceId = traceId != null ? Some.apply(traceId) : Option.empty(); - - // Arguments are unused by the runtime in case of Truffle. - Option[]> maybeArguments = Option.empty(); + // Set program environment. this.programEnvironment = - new ProgramEnvironment( - this.user, maybeArguments, scalaScopes, scalaOptions, maybeTraceId, Option.empty()); + ProgramEnvironment$.MODULE$.deserializeFromString( + env.getEnvironment().get("RAW_PROGRAM_ENVIRONMENT")); // The function registry holds snapi methods (top level functions). It is the data // structure that is used to extract a ref to a function from a piece of execute snapi. @@ -123,40 +82,82 @@ public ProgramEnvironment getProgramEnvironment() { } public String getTraceId() { - return traceId; + return programEnvironment.maybeTraceId().get(); } public InferrerService getInferrer() { - return language.getInferrer(getUser(), rawSettings); + return language.getInferrer(getUid(), rawSettings); } public OutputStream getOutput() { return output; } + public RawSettings getSettings() { + return rawSettings; + } + @CompilerDirectives.TruffleBoundary - public SourceContext getSourceContext() { - return language.getSourceContext(getUser(), rawSettings); + public Map getHttpHeaders(String name) { + scala.Option> maybeHttpHeaders = + programEnvironment.httpHeaders().get(name); + if (maybeHttpHeaders.isEmpty()) { + throw new RawTruffleRuntimeException("unknown http credential: " + name); + } + return JavaConverters.mapAsJavaMap(maybeHttpHeaders.get()); } - public RawSettings getSettings() { - return rawSettings; + @CompilerDirectives.TruffleBoundary + public boolean existsSecret(String key) { + return programEnvironment.secrets().contains(key); } @CompilerDirectives.TruffleBoundary - public Secret getSecret(String key) { - if (user == null) { - throw new RawTruffleRuntimeException("User not set"); + public String getSecret(String key) { + scala.Option maybeSecret = programEnvironment.secrets().get(key); + if (maybeSecret.isEmpty()) { + throw new RawTruffleRuntimeException("unknown secret: " + key); } - return getSourceContext().credentialsService().getSecret(user, key).get(); + return maybeSecret.get(); } - public AuthenticatedUser getUser() { - return user; + @CompilerDirectives.TruffleBoundary + public boolean existsJdbcCredential(String name) { + return programEnvironment.jdbcServers().contains(name); } + @CompilerDirectives.TruffleBoundary + public JdbcLocation getJdbcLocation(String name) { + scala.Option maybeJdbcLocation = programEnvironment.jdbcServers().get(name); + if (maybeJdbcLocation.isEmpty()) { + throw new RawTruffleRuntimeException("unknown database credential: " + name); + } + return maybeJdbcLocation.get(); + } + + @CompilerDirectives.TruffleBoundary + public boolean existsS3Credential(String bucket) { + return programEnvironment.s3Credentials().contains(bucket); + } + + @CompilerDirectives.TruffleBoundary + public S3Credential getS3Credential(String bucket) { + scala.Option maybeCred = programEnvironment.s3Credentials().get(bucket); + if (maybeCred.isEmpty()) { + throw new RawTruffleRuntimeException("unknown S3 bucket: " + bucket); + } + return maybeCred.get(); + } + + @CompilerDirectives.TruffleBoundary + public RawUid getUid() { + return programEnvironment.uid(); + } + + @CompilerDirectives.TruffleBoundary public String[] getScopes() { - return scopes; + Set javaScopes = JavaConverters.setAsJavaSet(programEnvironment.scopes()); + return javaScopes.toArray(new String[0]); } private static final TruffleLanguage.ContextReference REFERENCE = diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguage.java b/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguage.java index 690e801a2..61e3bb49e 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguage.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguage.java @@ -43,9 +43,8 @@ import raw.runtime.truffle.runtime.exceptions.RawTruffleValidationException; import raw.runtime.truffle.runtime.record.DuplicateKeyRecord; import raw.runtime.truffle.runtime.record.PureRecord; -import raw.sources.api.SourceContext; -import raw.utils.AuthenticatedUser; import raw.utils.RawSettings; +import raw.utils.RawUid; import scala.collection.JavaConverters; @TruffleLanguage.Registration( @@ -123,7 +122,7 @@ protected CallTarget parse(ParsingRequest request) throws Exception { ProgramContext programContext = new Rql2ProgramContext( context.getProgramEnvironment(), - getCompilerContext(context.getUser(), context.getSettings())); + getCompilerContext(context.getUid(), context.getSettings())); String source = request.getSource().getCharacters().toString(); @@ -238,15 +237,11 @@ protected Object getScope(RawContext context) { return context.getFunctionRegistry().asPolyglot(); } - public SourceContext getSourceContext(AuthenticatedUser user, RawSettings rawSettings) { - return languageCache.getSourceContext(user, rawSettings); - } - - public CompilerContext getCompilerContext(AuthenticatedUser user, RawSettings rawSettings) { + public CompilerContext getCompilerContext(RawUid user, RawSettings rawSettings) { return languageCache.getCompilerContext(user, rawSettings); } - public InferrerService getInferrer(AuthenticatedUser user, RawSettings rawSettings) { + public InferrerService getInferrer(RawUid user, RawSettings rawSettings) { return languageCache.getInferrer(user, rawSettings); } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguageCache.java b/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguageCache.java index 1aff4f202..44861a4e3 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguageCache.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/RawLanguageCache.java @@ -17,13 +17,10 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import raw.compiler.base.CompilerContext; -import raw.creds.api.CredentialsService; -import raw.creds.api.CredentialsServiceProvider; import raw.inferrer.api.InferrerService; import raw.inferrer.api.InferrerServiceProvider; -import raw.sources.api.SourceContext; -import raw.utils.AuthenticatedUser; import raw.utils.RawSettings; +import raw.utils.RawUid; import raw.utils.RawUtils; import scala.runtime.BoxedUnit; @@ -34,19 +31,14 @@ public class RawLanguageCache { private final Object activeContextsLock = new Object(); private final Set activeContexts = new HashSet(); - private final ConcurrentHashMap credentialsCache = - new ConcurrentHashMap<>(); - - private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); private static class Value { private final CompilerContext compilerContext; - private final SourceContext sourceContext; private final InferrerService inferrer; - Value(CompilerContext compilerContext, SourceContext sourceContext, InferrerService inferrer) { + Value(CompilerContext compilerContext, InferrerService inferrer) { this.compilerContext = compilerContext; - this.sourceContext = sourceContext; this.inferrer = inferrer; } @@ -54,41 +46,28 @@ public CompilerContext getCompilerContext() { return compilerContext; } - public SourceContext getSourceContext() { - return sourceContext; - } - public InferrerService getInferrer() { return inferrer; } } @CompilerDirectives.TruffleBoundary - private Value get(AuthenticatedUser user, RawSettings rawSettings) { - // Create services on-demand. - CredentialsService credentialsService = - credentialsCache.computeIfAbsent( - rawSettings, k -> CredentialsServiceProvider.apply(rawSettings)); + private Value get(RawUid user, RawSettings rawSettings) { return map.computeIfAbsent( user, k -> { - SourceContext sourceContext = new SourceContext(user, credentialsService, rawSettings); - InferrerService inferrer = InferrerServiceProvider.apply(sourceContext); + InferrerService inferrer = InferrerServiceProvider.apply(rawSettings); CompilerContext compilerContext = - new CompilerContext("rql2-truffle", user, inferrer, sourceContext, rawSettings); - return new Value(compilerContext, sourceContext, inferrer); + new CompilerContext("rql2-truffle", user, inferrer, rawSettings); + return new Value(compilerContext, inferrer); }); } - public SourceContext getSourceContext(AuthenticatedUser user, RawSettings rawSettings) { - return get(user, rawSettings).getSourceContext(); - } - - public CompilerContext getCompilerContext(AuthenticatedUser user, RawSettings rawSettings) { + public CompilerContext getCompilerContext(RawUid user, RawSettings rawSettings) { return get(user, rawSettings).getCompilerContext(); } - public InferrerService getInferrer(AuthenticatedUser user, RawSettings rawSettings) { + public InferrerService getInferrer(RawUid user, RawSettings rawSettings) { return get(user, rawSettings).getInferrer(); } @@ -104,7 +83,7 @@ public void releaseContext(RawContext context) { synchronized (activeContextsLock) { activeContexts.remove(context); if (activeContexts.isEmpty()) { - // Close all inferrer services and credential services. + // Close all inferrer services. map.values() .forEach( v -> { @@ -114,26 +93,8 @@ public void releaseContext(RawContext context) { return BoxedUnit.UNIT; }, true); - RawUtils.withSuppressNonFatalException( - () -> { - v.getSourceContext().credentialsService().stop(); - return BoxedUnit.UNIT; - }, - true); }); map.clear(); - credentialsCache - .values() - .forEach( - v -> { - RawUtils.withSuppressNonFatalException( - () -> { - v.stop(); - return BoxedUnit.UNIT; - }, - true); - }); - credentialsCache.clear(); } } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/aws_package/AwsV4SignedRequestNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/aws_package/AwsV4SignedRequestNode.java index 3379ae665..b3ebc310b 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/aws_package/AwsV4SignedRequestNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/aws_package/AwsV4SignedRequestNode.java @@ -17,6 +17,7 @@ import com.oracle.truffle.api.dsl.NodeChild; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.NodeInfo; +import java.net.HttpURLConnection; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; @@ -28,18 +29,23 @@ import java.time.format.DateTimeFormatter; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import raw.client.api.*; import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; import raw.runtime.truffle.RawLanguage; import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; import raw.runtime.truffle.runtime.list.ListNodes; import raw.runtime.truffle.runtime.list.ObjectList; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.runtime.record.RecordNodes; +import raw.sources.bytestream.http.HttpByteStreamLocation; +import raw.utils.RawSettings; +import scala.None$; +import scala.Option; +import scala.Some; import scala.Tuple2; -import scala.collection.immutable.HashMap; -import scala.collection.immutable.Map; -import scala.collection.immutable.VectorBuilder; +import scala.collection.mutable.ArrayBuilder; +import scala.reflect.ClassTag; +import scala.reflect.ClassTag$; @NodeInfo(shortName = "Aws.V4SignedRequest") @NodeChild("key") @@ -132,9 +138,12 @@ protected LocationObject doRequest( String amzdate = formatterWithTimeZone().format(t); String datestamp = getDateFormatter().format(t); + ClassTag> tupleClassTag = + (ClassTag>) (ClassTag) ClassTag$.MODULE$.apply(Tuple2.class); + // Task 1: create canonical request with all request settings: method, canonicalUri, // canonicalQueryString etc. - VectorBuilder> urlParamsVec = new VectorBuilder<>(); + ArrayBuilder> argsBuilder = new ArrayBuilder.ofRef(tupleClassTag); StringBuilder canonicalQueryBuilder = new StringBuilder(); Object urlParamsSorted = sortNode.execute(this, urlParams); @@ -153,7 +162,7 @@ protected LocationObject doRequest( getValueNode.execute(this, getNode.execute(this, urlParamsSorted, i), "_2"), StandardCharsets.UTF_8)) .append("&"); - urlParamsVec.$plus$eq( + argsBuilder.$plus$eq( new Tuple2<>( (String) getValueNode.execute(this, getNode.execute(this, urlParamsSorted, i), "_1"), (String) @@ -173,7 +182,8 @@ protected LocationObject doRequest( // always required. StringBuilder canonicalHeadersBuilder = new StringBuilder(); StringBuilder signedHeadersBuilder = new StringBuilder(); - VectorBuilder> headersParamsVec = new VectorBuilder<>(); + + ArrayBuilder> headersBuilder = new ArrayBuilder.ofRef(tupleClassTag); int headersSize = (int) sizeNode.execute(this, headers); // Adding space for host and "x-amz-date", "host" and "x-amz-security-token" if it is @@ -223,7 +233,7 @@ protected LocationObject doRequest( } for (int i = 0; i < sizeNode.execute(this, headers); i++) { - headersParamsVec.$plus$eq( + headersBuilder.$plus$eq( new Tuple2<>( getValueNode .execute(this, getNode.execute(this, headers, i), "_1") @@ -293,41 +303,40 @@ protected LocationObject doRequest( + "Signature=" + signature; - VectorBuilder> newHeaders = new VectorBuilder<>(); - newHeaders.$plus$eq(new Tuple2<>("x-amz-date", amzdate)); - newHeaders.$plus$eq(new Tuple2<>("Authorization", authorizationHeader)); + headersBuilder.$plus$eq(new Tuple2<>("x-amz-date", amzdate)); + headersBuilder.$plus$eq(new Tuple2<>("Authorization", authorizationHeader)); if (!sessionToken.isEmpty()) { - newHeaders.$plus$eq(new Tuple2<>("x-amz-security-token", sessionToken)); + headersBuilder.$plus$eq(new Tuple2<>("x-amz-security-token", sessionToken)); } - VectorBuilder> requestHeaders = - newHeaders.$plus$plus$eq(headersParamsVec.result()); - - // host is added automatically - Map map = new HashMap<>(); - map = - map.$plus( - Tuple2.apply(new LocationSettingKey("http-method"), new LocationStringSetting(method))); - map = - map.$plus( - Tuple2.apply( - new LocationSettingKey("http-args"), new LocationKVSetting(urlParamsVec.result()))); - map = - map.$plus( - Tuple2.apply( - new LocationSettingKey("http-headers"), - new LocationKVSetting(requestHeaders.result()))); - + Option maybeBody; if (!bodyString.isEmpty()) { - map = - map.$plus( - Tuple2.apply( - new LocationSettingKey("http-body-string"), - new LocationStringSetting(bodyString))); + maybeBody = new Some(bodyString.getBytes()); + } else { + maybeBody = None$.empty(); } String url = "https://" + host + "/" + path.replaceAll("^/+", ""); - return new LocationObject(url, map); + int[] expectedStatusArray = { + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_ACCEPTED, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_PARTIAL + }; + + RawSettings rawSettings = RawContext.get(this).getSettings(); + + HttpByteStreamLocation location = + new HttpByteStreamLocation( + url, + method, + (Tuple2[]) argsBuilder.result(), + (Tuple2[]) headersBuilder.result(), + maybeBody, + expectedStatusArray, + rawSettings); + + return new LocationObject(location, "aws:" + service + ":" + region + ":" + path); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/binary_package/BinaryReadNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/binary_package/BinaryReadNode.java index aaaa5f769..7dc8c73d0 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/binary_package/BinaryReadNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/binary_package/BinaryReadNode.java @@ -14,7 +14,6 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.*; -import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.NodeInfo; import java.io.IOException; import java.io.InputStream; @@ -26,7 +25,6 @@ import raw.runtime.truffle.runtime.primitives.ErrorObject; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "Binary.Read") @NodeChild(value = "binary") @@ -35,13 +33,10 @@ public abstract class BinaryReadNode extends ExpressionNode { @Specialization @TruffleBoundary - protected Object doExecute( - LocationObject locationObject, - @Bind("$node") Node thisNode, - @Cached(value = "getSourceContext(thisNode)", neverDefault = true) SourceContext context) { + protected Object doExecute(LocationObject locationObject) { InputStream stream = null; try { - stream = (new TruffleInputStream(locationObject, context)).getInputStream(); + stream = (new TruffleInputStream(locationObject)).getInputStream(); return new BinaryObject(stream.readAllBytes()); } catch (IOException | RawTruffleRuntimeException ex) { return new ErrorObject(ex.getMessage()); diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/environment_package/EnvironmentSecretNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/environment_package/EnvironmentSecretNode.java index f573ad194..48b2c4c11 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/environment_package/EnvironmentSecretNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/environment_package/EnvironmentSecretNode.java @@ -15,10 +15,9 @@ import com.oracle.truffle.api.dsl.*; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.NodeInfo; -import java.util.NoSuchElementException; -import raw.creds.api.Secret; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; import raw.runtime.truffle.runtime.generator.collection.StaticInitializers; import raw.runtime.truffle.runtime.primitives.ErrorObject; @@ -33,10 +32,9 @@ protected static Object doSecret( @Bind("$node") Node thisNode, @Cached(value = "getRawContext(thisNode)", neverDefault = true) RawContext context) { try { - Secret v = context.getSecret(key); - return v.value(); - } catch (NoSuchElementException e) { - return new ErrorObject("could not find secret " + key); + return context.getSecret(key); + } catch (RawTruffleRuntimeException e) { + return new ErrorObject(e.getMessage()); } } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/http_package/HttpReadNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/http_package/HttpReadNode.java index 36c270e02..69e20d6a2 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/http_package/HttpReadNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/http_package/HttpReadNode.java @@ -30,9 +30,7 @@ import raw.runtime.truffle.runtime.record.RecordNodes; import raw.runtime.truffle.tryable_nullable.TryableNullableNodes; import raw.sources.api.LocationException; -import raw.sources.api.SourceContext; import raw.sources.bytestream.http.HttpByteStreamLocation; -import raw.sources.bytestream.http.HttpByteStreamLocationBuilder; import raw.sources.bytestream.http.HttpResult; import scala.Tuple2; import scala.collection.IndexedSeq; @@ -52,12 +50,9 @@ protected static Object doRead( @Cached(inline = true) TryableNullableNodes.IsNullNode isNullNode, @Cached(inline = true) ListNodes.SizeNode sizeNode, @Cached(inline = true) ListNodes.GetNode getNode, - @Cached(inline = true) RecordNodes.AddPropNode addPropNode, - @Cached(value = "getSourceContext(thisNode)", neverDefault = true) SourceContext context) { + @Cached(inline = true) RecordNodes.AddPropNode addPropNode) { try { - HttpByteStreamLocationBuilder builder = new HttpByteStreamLocationBuilder(); - HttpByteStreamLocation location = - builder.build(locationObject.getLocationDescription(), context); + HttpByteStreamLocation location = locationObject.getHttpByteStreamLocation(); HttpResult result = location.getHttpResult(); Object record = RawLanguage.get(thisNode).createPureRecord(); @@ -67,15 +62,10 @@ protected static Object doRead( statuses[i] = (int) getNode.execute(thisNode, statusListOption, i); } if (Arrays.stream(statuses).noneMatch(status -> status == result.status())) { - String method = - locationObject - .getLocationDescription() - .getStringSetting("http-method") - .getOrElse(() -> "get"); return new ErrorObject( String.format( "HTTP %s failed, got %d, expected %s", - method.toUpperCase(), + location.method().toUpperCase(), result.status(), String.join( ",", diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQueryNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/jdbc/JdbcQueryNode.java similarity index 86% rename from snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQueryNode.java rename to snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/jdbc/JdbcQueryNode.java index c5860cc13..5dabf997e 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQueryNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/jdbc/JdbcQueryNode.java @@ -10,7 +10,7 @@ * licenses/APL.txt. */ -package raw.runtime.truffle.ast.io.jdbc; +package raw.runtime.truffle.ast.expressions.builtin.jdbc; import com.oracle.truffle.api.RootCallTarget; import com.oracle.truffle.api.frame.VirtualFrame; @@ -21,7 +21,7 @@ import raw.runtime.truffle.runtime.exceptions.rdbms.JdbcExceptionHandler; import raw.runtime.truffle.runtime.iterable.sources.JdbcQueryCollection; import raw.runtime.truffle.runtime.primitives.LocationObject; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; @NodeInfo(shortName = "Jdbc.Query") public class JdbcQueryNode extends ExpressionNode { @@ -30,7 +30,6 @@ public class JdbcQueryNode extends ExpressionNode { @Child private ExpressionNode queryExp; private final RootCallTarget makeRowCallTarget; private final JdbcExceptionHandler exceptionHandler; - private final SourceContext context = RawContext.get(this).getSourceContext(); public JdbcQueryNode( ExpressionNode locationExp, @@ -45,8 +44,10 @@ public JdbcQueryNode( @Override public Object executeGeneric(VirtualFrame virtualFrame) { + RawSettings rawSettings = RawContext.get(this).getSettings(); LocationObject dbLocation = (LocationObject) locationExp.executeGeneric(virtualFrame); String query = (String) this.queryExp.executeGeneric(virtualFrame); - return new JdbcQueryCollection(dbLocation, query, context, makeRowCallTarget, exceptionHandler); + return new JdbcQueryCollection( + dbLocation, query, rawSettings, makeRowCallTarget, exceptionHandler); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationBuildNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationBuildNode.java deleted file mode 100644 index 7abf1f91b..000000000 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationBuildNode.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.runtime.truffle.ast.expressions.builtin.location_package; - -import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; -import com.oracle.truffle.api.frame.VirtualFrame; -import com.oracle.truffle.api.interop.InteropLibrary; -import com.oracle.truffle.api.interop.InvalidArrayIndexException; -import com.oracle.truffle.api.interop.UnknownIdentifierException; -import com.oracle.truffle.api.interop.UnsupportedMessageException; -import com.oracle.truffle.api.nodes.ExplodeLoop; -import com.oracle.truffle.api.nodes.NodeInfo; -import java.time.Duration; -import raw.client.api.*; -import raw.compiler.rql2.source.Rql2IntType; -import raw.compiler.rql2.source.Rql2ListType; -import raw.compiler.rql2.source.Rql2TypeWithProperties; -import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.ast.TypeGuards; -import raw.runtime.truffle.ast.expressions.builtin.temporals.interval_package.IntervalNodes; -import raw.runtime.truffle.ast.expressions.builtin.temporals.interval_package.IntervalNodesFactory; -import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; -import raw.runtime.truffle.runtime.list.ListNodes; -import raw.runtime.truffle.runtime.list.ListNodesFactory; -import raw.runtime.truffle.runtime.primitives.*; -import scala.Tuple2; -import scala.collection.immutable.HashMap; -import scala.collection.immutable.Map; -import scala.collection.immutable.VectorBuilder; - -@NodeInfo(shortName = "Location.Build") -public class LocationBuildNode extends ExpressionNode { - - private final String[] keys; - @Child private InteropLibrary interops = InteropLibrary.getFactory().createDispatched(3); - @Child private ExpressionNode url; - @Child private ListNodes.SizeNode sizeNode = ListNodesFactory.SizeNodeGen.create(); - - @Child private ListNodes.GetNode getNode = ListNodesFactory.GetNodeGen.create(); - - @Children private final ExpressionNode[] values; - - private final Rql2TypeWithProperties[] types; - - public LocationBuildNode( - ExpressionNode url, String[] keys, ExpressionNode[] values, Rql2TypeWithProperties[] types) { - assert values.length == keys.length; - assert values.length == types.length; - this.url = url; - this.keys = keys; - this.values = values; - this.types = types; - } - - @Override - @ExplodeLoop - public Object executeGeneric(VirtualFrame frame) { - Map map = new HashMap<>(); - String url = (String) this.url.executeGeneric(frame); - for (int i = 0; i < this.keys.length; i++) { - Object value = this.values[i].executeGeneric(frame); - map = addToMap(map, this.keys[i], value, i); - } - return new LocationObject(url, map); - } - - @TruffleBoundary - private Map addToMap( - Map map, String key, Object value, int index) { - return map.$plus( - Tuple2.apply(new LocationSettingKey(key), buildLocationSettingValue(value, types[index]))); - } - - @TruffleBoundary - private LocationSettingValue buildLocationSettingValue( - Object value, Rql2TypeWithProperties type) { - try { - IntervalNodes.IntervalToMillisStaticNode toMillisNode = - IntervalNodesFactory.IntervalToMillisStaticNodeGen.getUncached(); - if (TypeGuards.isIntKind(type)) { - return new LocationIntSetting((Integer) value); - } else if (TypeGuards.isStringKind(type)) { - return new LocationStringSetting((String) value); - } else if (TypeGuards.isByteKind(type)) { - byte[] bytes = (byte[]) value; - VectorBuilder vec = new VectorBuilder<>(); - for (byte aByte : bytes) { - vec = vec.$plus$eq(aByte); - } - return new LocationBinarySetting(vec.result()); - } else if (TypeGuards.isBooleanKind(type)) { - return new LocationBooleanSetting((Boolean) value); - } else if (TypeGuards.isIntervalKind(type)) { - return new LocationDurationSetting( - Duration.ofMillis( - (IntervalNodesFactory.IntervalToMillisStaticNodeGen.getUncached() - .execute(this, (IntervalObject) value)))); - } else if (TypeGuards.isListKind(type) - && ((Rql2ListType) type).innerType() instanceof Rql2IntType) { - int[] ints = new int[(int) sizeNode.execute(this, value)]; - for (int i = 0; i < ints.length; i++) { - ints[i] = (int) getNode.execute(this, value, i); - } - return new LocationIntArraySetting(ints); - } else if (TypeGuards.isListKind(type)) { - VectorBuilder> vec = new VectorBuilder<>(); - int size = (int) sizeNode.execute(this, value); - for (int i = 0; i < size; i++) { - Object record = getNode.execute(this, value, i); - Object keys = interops.getMembers(record); - Object key = interops.readMember(record, (String) interops.readArrayElement(keys, 0)); - Object val = interops.readMember(record, (String) interops.readArrayElement(keys, 1)); - // ignore entries where key or val is null - if (key != NullObject.INSTANCE && val != NullObject.INSTANCE) - vec = vec.$plus$eq(Tuple2.apply((String) key, (String) val)); - } - return new LocationKVSetting(vec.result()); - } else if (TypeGuards.isBinaryKind(type)) { - VectorBuilder vec = new VectorBuilder<>(); - for (byte aByte : ((BinaryObject) value).getBytes()) { - vec = vec.$plus$eq(aByte); - } - return new LocationBinarySetting(vec.result()); - } else { - throw new RawTruffleInternalErrorException(); - } - } catch (UnsupportedMessageException - | UnknownIdentifierException - | InvalidArrayIndexException e) { - throw new RawTruffleInternalErrorException(e, this); - } - } -} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationDescribeNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationDescribeNode.java index 1773394ab..bcfea1f40 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationDescribeNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationDescribeNode.java @@ -54,7 +54,7 @@ protected Object doDescribe( InputFormatDescriptor descriptor = inferrer.infer( AutoInferrerProperties.apply( - locationObject.getLocationDescription(), + locationObject.getLocation(), sampleSize == Integer.MAX_VALUE ? Some.empty() : Some.apply(sampleSize))); String format = ""; diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java new file mode 100644 index 000000000..d679334ed --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromHttpNode.java @@ -0,0 +1,211 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidArrayIndexException; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.nodes.NodeInfo; +import java.net.HttpURLConnection; +import java.util.Base64; +import java.util.Map; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; +import raw.runtime.truffle.runtime.list.ListNodes; +import raw.runtime.truffle.runtime.list.ListNodesFactory; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.bytestream.http.HttpByteStreamLocation; +import raw.utils.RawSettings; +import scala.None$; +import scala.Option; +import scala.Some; +import scala.Tuple2; +import scala.collection.mutable.ArrayBuilder; +import scala.reflect.ClassTag; +import scala.reflect.ClassTag$; + +@NodeInfo(shortName = "Location.FromHttp") +public class LocationFromHttpNode extends ExpressionNode { + + private String method; + @Child private ExpressionNode url; + @Child private ExpressionNode bodyString; + @Child private ExpressionNode bodyBinary; + @Child private ExpressionNode authCredentialName; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + @Child private ExpressionNode args; + @Child private ExpressionNode headers; + @Child private ExpressionNode expectedStatus; + + @Child private InteropLibrary argsInterops = InteropLibrary.getFactory().createDispatched(3); + @Child private ListNodes.SizeNode argsSizeNode = ListNodesFactory.SizeNodeGen.create(); + @Child private ListNodes.GetNode argsGetNode = ListNodesFactory.GetNodeGen.create(); + + @Child private InteropLibrary headersInterops = InteropLibrary.getFactory().createDispatched(3); + @Child private ListNodes.SizeNode headersSizeNode = ListNodesFactory.SizeNodeGen.create(); + @Child private ListNodes.GetNode headersGetNode = ListNodesFactory.GetNodeGen.create(); + + @Child private ListNodes.SizeNode expectedStatusSizeNode = ListNodesFactory.SizeNodeGen.create(); + @Child private ListNodes.GetNode expectedStatusGetNode = ListNodesFactory.GetNodeGen.create(); + + public LocationFromHttpNode( + String method, + ExpressionNode url, + ExpressionNode bodyString, + ExpressionNode bodyBinary, + ExpressionNode authCredentialName, + ExpressionNode username, + ExpressionNode password, + ExpressionNode args, + ExpressionNode headers, + ExpressionNode expectedStatus) { + this.method = method; + this.url = url; + this.bodyString = bodyString; + this.bodyBinary = bodyBinary; + this.authCredentialName = authCredentialName; + this.username = username; + this.password = password; + this.args = args; + this.headers = headers; + this.expectedStatus = expectedStatus; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + try { + String url = (String) this.url.executeGeneric(frame); + + // Build body + Option maybeBody; + if (this.bodyString != null) { + maybeBody = new Some(((String) this.bodyString.executeGeneric(frame)).getBytes()); + } else if (this.bodyBinary != null) { + maybeBody = new Some(((BinaryObject) this.bodyBinary.executeGeneric(frame)).getBytes()); + } else { + maybeBody = None$.empty(); + } + + // Build args vector + ClassTag> tupleClassTag = + (ClassTag>) (ClassTag) ClassTag$.MODULE$.apply(Tuple2.class); + ArrayBuilder> argsBuilder = new ArrayBuilder.ofRef(tupleClassTag); + if (this.args != null) { + Object value = this.args.executeGeneric(frame); + int size = (int) this.argsSizeNode.execute(this, value); + for (int i = 0; i < size; i++) { + Object record = this.argsGetNode.execute(this, value, i); + Object keys = this.argsInterops.getMembers(record); + Object key = + this.argsInterops.readMember( + record, (String) this.argsInterops.readArrayElement(keys, 0)); + Object val = + this.argsInterops.readMember( + record, (String) this.argsInterops.readArrayElement(keys, 1)); + // ignore entries where key or val is null + if (key != NullObject.INSTANCE && val != NullObject.INSTANCE) { + argsBuilder = + (ArrayBuilder>) + argsBuilder.$plus$eq(Tuple2.apply((String) key, (String) val)); + } + } + } + + // Build headers vector + ArrayBuilder> headersBuilder = new ArrayBuilder.ofRef(tupleClassTag); + if (this.headers != null) { + Object value = this.headers.executeGeneric(frame); + int size = (int) this.headersSizeNode.execute(this, value); + for (int i = 0; i < size; i++) { + Object record = this.headersGetNode.execute(this, value, i); + Object keys = this.headersInterops.getMembers(record); + Object key = + this.headersInterops.readMember( + record, (String) this.headersInterops.readArrayElement(keys, 0)); + Object val = + this.headersInterops.readMember( + record, (String) this.headersInterops.readArrayElement(keys, 1)); + // ignore entries where key or val is null + if (key != NullObject.INSTANCE && val != NullObject.INSTANCE) { + headersBuilder = + (ArrayBuilder>) + headersBuilder.$plus$eq(Tuple2.apply((String) key, (String) val)); + } + } + } + + // Append Authorization header if username and password are provided + if (this.username != null && this.password != null) { + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + headersBuilder = + (ArrayBuilder>) + headersBuilder.$plus$eq( + Tuple2.apply( + "Authorization", + "Basic " + + Base64.getEncoder() + .encodeToString((username + ":" + password).getBytes()))); + } + + // Append any additional headers related to the authentication (if credential name is defined) + if (this.authCredentialName != null) { + String authCredentialName = (String) this.authCredentialName.executeGeneric(frame); + Map credHeaders = RawContext.get(this).getHttpHeaders(authCredentialName); + for (Map.Entry entry : credHeaders.entrySet()) { + headersBuilder = + (ArrayBuilder>) + headersBuilder.$plus$eq(Tuple2.apply(entry.getKey(), entry.getValue())); + } + } + + // Build expected status vector + int[] expectedStatusArray = { + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_ACCEPTED, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_PARTIAL + }; + if (this.expectedStatus != null) { + Object value = this.expectedStatus.executeGeneric(frame); + int size = (int) this.expectedStatusSizeNode.execute(this, value); + expectedStatusArray = new int[size]; + for (int i = 0; i < size; i++) { + expectedStatusArray[i] = (int) this.expectedStatusGetNode.execute(this, value, i); + } + } + + RawSettings rawSettings = RawContext.get(this).getSettings(); + + HttpByteStreamLocation location = + new HttpByteStreamLocation( + url, + this.method, + (Tuple2[]) argsBuilder.result(), + (Tuple2[]) headersBuilder.result(), + maybeBody, + expectedStatusArray, + rawSettings); + + return new LocationObject(location, url); + } catch (UnsupportedMessageException + | InvalidArrayIndexException + | UnknownIdentifierException e) { + throw new RawTruffleInternalErrorException(e, this); + } + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLCredentialNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLCredentialNode.java new file mode 100644 index 000000000..bf9aa0c93 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLCredentialNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.JdbcLocation; +import raw.compiler.rql2.api.LocationDescription$; +import raw.compiler.rql2.api.MySqlServerLocationDescription; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.mysql.MySqlServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromMySQLCredential") +public class LocationFromMySQLCredentialNode extends ExpressionNode { + + @Child private ExpressionNode credentialName; + + public LocationFromMySQLCredentialNode(ExpressionNode credentialName) { + this.credentialName = credentialName; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + RawContext context = RawContext.get(this); + + String credentialName = (String) this.credentialName.executeGeneric(frame); + JdbcLocation l = context.getJdbcLocation(credentialName); + JdbcServerLocation location = getJdbcServerLocation(l, context.getSettings()); + + return new LocationObject(location, "mysql:" + credentialName); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation(JdbcLocation l, RawSettings rawSettings) { + MySqlServerLocationDescription d = + (MySqlServerLocationDescription) LocationDescription$.MODULE$.toLocationDescription(l); + return new MySqlServerLocation( + d.host(), d.port(), d.dbName(), d.username(), d.password(), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLNode.java new file mode 100644 index 000000000..3730c9e41 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromMySQLNode.java @@ -0,0 +1,67 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.mysql.MySqlServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromMySQL") +public class LocationFromMySQLNode extends ExpressionNode { + + @Child private ExpressionNode host; + @Child private ExpressionNode port; + @Child private ExpressionNode db; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + + public LocationFromMySQLNode( + ExpressionNode host, + ExpressionNode port, + ExpressionNode db, + ExpressionNode username, + ExpressionNode password) { + this.host = host; + this.port = port; + this.db = db; + this.username = username; + this.password = password; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String host = (String) this.host.executeGeneric(frame); + int port = (int) this.port.executeGeneric(frame); + String db = (String) this.db.executeGeneric(frame); + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + + JdbcServerLocation location = + getJdbcServerLocation( + host, port, db, username, password, RawContext.get(this).getSettings()); + + return new LocationObject(location, "mysql:" + db); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation( + String host, int port, String db, String username, String password, RawSettings rawSettings) { + return new MySqlServerLocation(host, port, db, username, password, rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleCredentialNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleCredentialNode.java new file mode 100644 index 000000000..d26242d83 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleCredentialNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.JdbcLocation; +import raw.compiler.rql2.api.LocationDescription$; +import raw.compiler.rql2.api.OracleServerLocationDescription; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.oracle.OracleServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromOracleCredential") +public class LocationFromOracleCredentialNode extends ExpressionNode { + + @Child private ExpressionNode credentialName; + + public LocationFromOracleCredentialNode(ExpressionNode credentialName) { + this.credentialName = credentialName; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + RawContext context = RawContext.get(this); + + String credentialName = (String) this.credentialName.executeGeneric(frame); + JdbcLocation l = context.getJdbcLocation(credentialName); + JdbcServerLocation location = getJdbcServerLocation(l, context.getSettings()); + + return new LocationObject(location, "oracle:" + credentialName); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation(JdbcLocation l, RawSettings rawSettings) { + OracleServerLocationDescription d = + (OracleServerLocationDescription) LocationDescription$.MODULE$.toLocationDescription(l); + return new OracleServerLocation( + d.host(), d.port(), d.dbName(), d.username(), d.password(), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleNode.java new file mode 100644 index 000000000..0a60a6e96 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromOracleNode.java @@ -0,0 +1,67 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.oracle.OracleServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromOracle") +public class LocationFromOracleNode extends ExpressionNode { + + @Child private ExpressionNode host; + @Child private ExpressionNode port; + @Child private ExpressionNode db; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + + public LocationFromOracleNode( + ExpressionNode host, + ExpressionNode port, + ExpressionNode db, + ExpressionNode username, + ExpressionNode password) { + this.host = host; + this.port = port; + this.db = db; + this.username = username; + this.password = password; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String host = (String) this.host.executeGeneric(frame); + int port = (int) this.port.executeGeneric(frame); + String db = (String) this.db.executeGeneric(frame); + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + + JdbcServerLocation location = + getJdbcServerLocation( + host, port, db, username, password, RawContext.get(this).getSettings()); + + return new LocationObject(location, "oracle:" + db); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation( + String host, int port, String db, String username, String password, RawSettings rawSettings) { + return new OracleServerLocation(host, port, db, username, password, rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLCredentialNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLCredentialNode.java new file mode 100644 index 000000000..fe2bb1e68 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLCredentialNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.JdbcLocation; +import raw.compiler.rql2.api.LocationDescription$; +import raw.compiler.rql2.api.PostgresqlServerLocationDescription; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.pgsql.PostgresqlServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromPostgreSQLCredential") +public class LocationFromPostgreSQLCredentialNode extends ExpressionNode { + + @Child private ExpressionNode credentialName; + + public LocationFromPostgreSQLCredentialNode(ExpressionNode credentialName) { + this.credentialName = credentialName; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + RawContext context = RawContext.get(this); + + String credentialName = (String) this.credentialName.executeGeneric(frame); + JdbcLocation l = context.getJdbcLocation(credentialName); + JdbcServerLocation location = getJdbcServerLocation(l, context.getSettings()); + + return new LocationObject(location, "pgsql:" + credentialName); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation(JdbcLocation l, RawSettings rawSettings) { + PostgresqlServerLocationDescription d = + (PostgresqlServerLocationDescription) LocationDescription$.MODULE$.toLocationDescription(l); + return new PostgresqlServerLocation( + d.host(), d.port(), d.dbName(), d.username(), d.password(), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLNode.java new file mode 100644 index 000000000..45d8cc96a --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromPostgreSQLNode.java @@ -0,0 +1,67 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.pgsql.PostgresqlServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromPostgreSQL") +public class LocationFromPostgreSQLNode extends ExpressionNode { + + @Child private ExpressionNode host; + @Child private ExpressionNode port; + @Child private ExpressionNode db; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + + public LocationFromPostgreSQLNode( + ExpressionNode host, + ExpressionNode port, + ExpressionNode db, + ExpressionNode username, + ExpressionNode password) { + this.host = host; + this.port = port; + this.db = db; + this.username = username; + this.password = password; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String host = (String) this.host.executeGeneric(frame); + int port = (int) this.port.executeGeneric(frame); + String db = (String) this.db.executeGeneric(frame); + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + + JdbcServerLocation location = + getJdbcServerLocation( + host, port, db, username, password, RawContext.get(this).getSettings()); + + return new LocationObject(location, "pgsql:" + db); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation( + String host, int port, String db, String username, String password, RawSettings rawSettings) { + return new PostgresqlServerLocation(host, port, db, username, password, rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java new file mode 100644 index 000000000..92b920212 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromS3Node.java @@ -0,0 +1,99 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.S3Credential; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.filesystem.s3.S3Path; +import scala.None$; +import scala.Option; +import scala.Some; + +@NodeInfo(shortName = "Location.FromS3") +public class LocationFromS3Node extends ExpressionNode { + + @Child private ExpressionNode url; + @Child private ExpressionNode accessKey; + @Child private ExpressionNode secretKey; + @Child private ExpressionNode region; + + public LocationFromS3Node( + ExpressionNode url, + ExpressionNode accessKey, + ExpressionNode secretKey, + ExpressionNode region) { + this.url = url; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.region = region; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String url = (String) this.url.executeGeneric(frame); + + // Parse S3 URL to obtain S3 bucket and path + if (!url.startsWith("s3://")) { + throw new RawTruffleRuntimeException("invalid S3 URL format: " + url); + } + + // Remove the "s3://" prefix + String urlWithoutPrefix = url.substring(5); + + // Split the remaining part into bucket and path + int slashIndex = urlWithoutPrefix.indexOf('/'); + if (slashIndex == -1) { + throw new IllegalArgumentException("invalid S3 URL format: " + url); + } + + String bucket = urlWithoutPrefix.substring(0, slashIndex); + String path = urlWithoutPrefix.substring(slashIndex + 1); + + // The docs say: + // "If the S3 bucket is not registered in the credentials storage, then the region, accessKey + // and secretKey must be provided as arguments." + RawContext context = RawContext.get(this); + S3Path location; + + if (RawContext.get(this).existsS3Credential(bucket)) { + S3Credential cred = context.getS3Credential(bucket); + location = + new S3Path( + bucket, + cred.region(), + cred.accessKey(), + cred.secretKey(), + path, + context.getSettings()); + } else { + // We actually do NOT throw an exception if the accessKey is not passed. + // Instead, we go without it, which triggers anonymous access to the S3 bucket. + Option maybeAccessKey = + (this.accessKey != null) ? new Some(this.accessKey.executeGeneric(frame)) : None$.empty(); + Option maybeSecretKey = + (this.secretKey != null) ? new Some(this.secretKey.executeGeneric(frame)) : None$.empty(); + Option maybeRegion = + (this.region != null) ? new Some(this.region.executeGeneric(frame)) : None$.empty(); + location = + new S3Path( + bucket, maybeRegion, maybeAccessKey, maybeSecretKey, path, context.getSettings()); + } + + return new LocationObject(location, url); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerCredentialNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerCredentialNode.java new file mode 100644 index 000000000..7364846db --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerCredentialNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.JdbcLocation; +import raw.compiler.rql2.api.LocationDescription$; +import raw.compiler.rql2.api.SqlServerServerLocationDescription; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.sqlserver.SqlServerServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromSQLServerCredential") +public class LocationFromSQLServerCredentialNode extends ExpressionNode { + + @Child private ExpressionNode credentialName; + + public LocationFromSQLServerCredentialNode(ExpressionNode credentialName) { + this.credentialName = credentialName; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + RawContext context = RawContext.get(this); + + String credentialName = (String) this.credentialName.executeGeneric(frame); + JdbcLocation l = context.getJdbcLocation(credentialName); + JdbcServerLocation location = getJdbcServerLocation(l, context.getSettings()); + + return new LocationObject(location, "sqlserver:" + credentialName); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation(JdbcLocation l, RawSettings rawSettings) { + SqlServerServerLocationDescription d = + (SqlServerServerLocationDescription) LocationDescription$.MODULE$.toLocationDescription(l); + return new SqlServerServerLocation( + d.host(), d.port(), d.dbName(), d.username(), d.password(), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerNode.java new file mode 100644 index 000000000..0abcaa2b0 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSQLServerNode.java @@ -0,0 +1,67 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.LocationObject; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.sqlserver.SqlServerServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromSQLServer") +public class LocationFromSQLServerNode extends ExpressionNode { + + @Child private ExpressionNode host; + @Child private ExpressionNode port; + @Child private ExpressionNode db; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + + public LocationFromSQLServerNode( + ExpressionNode host, + ExpressionNode port, + ExpressionNode db, + ExpressionNode username, + ExpressionNode password) { + this.host = host; + this.port = port; + this.db = db; + this.username = username; + this.password = password; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String host = (String) this.host.executeGeneric(frame); + int port = (int) this.port.executeGeneric(frame); + String db = (String) this.db.executeGeneric(frame); + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + + JdbcServerLocation location = + getJdbcServerLocation( + host, port, db, username, password, RawContext.get(this).getSettings()); + + return new LocationObject(location, "sqlserver:" + db); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation( + String host, int port, String db, String username, String password, RawSettings rawSettings) { + return new SqlServerServerLocation(host, port, db, username, password, rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeCredentialNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeCredentialNode.java new file mode 100644 index 000000000..0b2e17927 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeCredentialNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.client.api.JdbcLocation; +import raw.compiler.rql2.api.LocationDescription$; +import raw.compiler.rql2.api.SnowflakeServerLocationDescription; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.snowflake.SnowflakeServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromSnowflakeCredential") +public class LocationFromSnowflakeCredentialNode extends ExpressionNode { + + @Child private ExpressionNode credentialName; + + public LocationFromSnowflakeCredentialNode(ExpressionNode credentialName) { + this.credentialName = credentialName; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + RawContext context = RawContext.get(this); + + String credentialName = (String) this.credentialName.executeGeneric(frame); + JdbcLocation l = context.getJdbcLocation(credentialName); + JdbcServerLocation location = getJdbcServerLocation(l, context.getSettings()); + + return new LocationObject(location, "snowflake:" + credentialName); + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation(JdbcLocation l, RawSettings rawSettings) { + SnowflakeServerLocationDescription d = + (SnowflakeServerLocationDescription) LocationDescription$.MODULE$.toLocationDescription(l); + return new SnowflakeServerLocation( + d.dbName(), d.username(), d.password(), d.accountIdentifier(), d.parameters(), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeNode.java new file mode 100644 index 000000000..a32228041 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromSnowflakeNode.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidArrayIndexException; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.nodes.NodeInfo; +import java.util.HashMap; +import java.util.Map; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; +import raw.runtime.truffle.runtime.list.ListNodes; +import raw.runtime.truffle.runtime.list.ListNodesFactory; +import raw.runtime.truffle.runtime.primitives.*; +import raw.sources.jdbc.api.JdbcServerLocation; +import raw.sources.jdbc.snowflake.SnowflakeServerLocation; +import raw.utils.RawSettings; + +@NodeInfo(shortName = "Location.FromSnowflake") +public class LocationFromSnowflakeNode extends ExpressionNode { + + @Child private ExpressionNode db; + @Child private ExpressionNode username; + @Child private ExpressionNode password; + @Child private ExpressionNode accountID; + @Child private ExpressionNode options; + + @Child private InteropLibrary interops = InteropLibrary.getFactory().createDispatched(3); + @Child private ListNodes.SizeNode sizeNode = ListNodesFactory.SizeNodeGen.create(); + @Child private ListNodes.GetNode getNode = ListNodesFactory.GetNodeGen.create(); + + public LocationFromSnowflakeNode( + ExpressionNode db, + ExpressionNode username, + ExpressionNode password, + ExpressionNode accountID, + ExpressionNode options) { + this.db = db; + this.username = username; + this.password = password; + this.accountID = accountID; + this.options = options; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + try { + String db = (String) this.db.executeGeneric(frame); + String username = (String) this.username.executeGeneric(frame); + String password = (String) this.password.executeGeneric(frame); + String accountID = (String) this.accountID.executeGeneric(frame); + + // Build args vector + Map parameters = new HashMap<>(); + if (this.options != null) { + Object value = this.options.executeGeneric(frame); + int size = (int) sizeNode.execute(this, value); + for (int i = 0; i < size; i++) { + Object record = getNode.execute(this, value, i); + Object keys = interops.getMembers(record); + Object key = interops.readMember(record, (String) interops.readArrayElement(keys, 0)); + Object val = interops.readMember(record, (String) interops.readArrayElement(keys, 1)); + // ignore entries where key or val is null + if (key != NullObject.INSTANCE && val != NullObject.INSTANCE) { + parameters.put((String) key, (String) val); + } + } + } + + JdbcServerLocation location = + getJdbcServerLocation( + db, username, password, accountID, parameters, RawContext.get(this).getSettings()); + + return new LocationObject(location, "snowflake:" + db); + } catch (UnsupportedMessageException + | InvalidArrayIndexException + | UnknownIdentifierException e) { + throw new RawTruffleInternalErrorException(e, this); + } + } + + @CompilerDirectives.TruffleBoundary + public JdbcServerLocation getJdbcServerLocation( + String db, + String username, + String password, + String accountID, + Map parameters, + RawSettings rawSettings) { + return new SnowflakeServerLocation(db, username, password, accountID, parameters, rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromStringNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromStringNode.java new file mode 100644 index 000000000..199ec2ef9 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationFromStringNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.builtin.location_package; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.nodes.NodeInfo; +import raw.compiler.rql2.api.LocationDescription$; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; +import raw.runtime.truffle.runtime.primitives.LocationObject; +import raw.sources.api.Location; +import scala.util.Either; + +@NodeInfo(shortName = "Location.FromString") +public class LocationFromStringNode extends ExpressionNode { + + @Child private ExpressionNode url; + + public LocationFromStringNode(ExpressionNode url) { + this.url = url; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + String url = (String) this.url.executeGeneric(frame); + + RawContext context = RawContext.get(this); + + Location location = getLocationFromUrl(url, context); + return new LocationObject(location, url); + } + + @CompilerDirectives.TruffleBoundary + private Location getLocationFromUrl(String url, RawContext context) { + Either maybeLocation = + LocationDescription$.MODULE$.urlToLocation( + url, context.getProgramEnvironment(), context.getSettings()); + if (maybeLocation.isLeft()) { + throw new RawTruffleRuntimeException(maybeLocation.left().get()); + } + return maybeLocation.right().get(); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLlNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLlNode.java index d22e72d23..54aa24546 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLlNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLlNode.java @@ -19,8 +19,8 @@ import com.oracle.truffle.api.nodes.NodeInfo; import java.time.LocalDateTime; import java.time.ZoneOffset; +import raw.compiler.rql2.api.LocationDescription$; import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.RawContext; import raw.runtime.truffle.RawLanguage; import raw.runtime.truffle.runtime.list.ObjectList; import raw.runtime.truffle.runtime.list.StringList; @@ -29,7 +29,6 @@ import raw.runtime.truffle.runtime.primitives.NullObject; import raw.runtime.truffle.runtime.primitives.TimestampObject; import raw.runtime.truffle.runtime.record.RecordNodes; -import raw.sources.api.SourceContext; import raw.sources.filesystem.api.*; import raw.utils.RawException; import scala.Tuple2; @@ -43,9 +42,7 @@ public abstract class LocationLlNode extends ExpressionNode { protected Object doLl( LocationObject locationObject, @Cached(inline = true) RecordNodes.AddPropNode addPropNode) { try { - SourceContext context = RawContext.get(this).getSourceContext(); - FileSystemLocation fs = - FileSystemLocationProvider.build(locationObject.getLocationDescription(), context); + FileSystemLocation fs = locationObject.getFileSystemLocation(); IndexedSeq> values = fs.lsWithMetadata().toIndexedSeq(); int size = values.size(); @@ -54,7 +51,12 @@ protected Object doLl( for (int i = 0; i < size; i++) { Object topRecord = RawLanguage.get(this).createPureRecord(); Object metadata = RawLanguage.get(this).createPureRecord(); - addPropNode.execute(this, topRecord, "url", values.apply(i)._1.rawUri(), false); + addPropNode.execute( + this, + topRecord, + "url", + LocationDescription$.MODULE$.locationToPublicUrl(values.apply(i)._1), + false); if (values.apply(i)._2 instanceof DirectoryMetadata) { DirectoryMetadata directoryMetadata = (DirectoryMetadata) values.apply(i)._2; if (directoryMetadata.modifiedInstant().isDefined()) { diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLsNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLsNode.java index 541f14421..c3dde3c89 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLsNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/location_package/LocationLsNode.java @@ -16,15 +16,12 @@ import com.oracle.truffle.api.dsl.NodeChild; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.NodeInfo; +import raw.compiler.rql2.api.LocationDescription$; import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.RawContext; import raw.runtime.truffle.runtime.list.StringList; import raw.runtime.truffle.runtime.primitives.ErrorObject; import raw.runtime.truffle.runtime.primitives.LocationObject; -import raw.sources.api.Location; -import raw.sources.api.SourceContext; import raw.sources.filesystem.api.FileSystemLocation; -import raw.sources.filesystem.api.FileSystemLocationProvider; import raw.utils.RawException; import scala.collection.IndexedSeq; @@ -35,10 +32,9 @@ public abstract class LocationLsNode extends ExpressionNode { @TruffleBoundary protected Object doLs(LocationObject locationObject) { try { - SourceContext context = RawContext.get(this).getSourceContext(); - FileSystemLocation fs = - FileSystemLocationProvider.build(locationObject.getLocationDescription(), context); - IndexedSeq values = fs.ls().map(Location::rawUri).toIndexedSeq(); + FileSystemLocation fs = locationObject.getFileSystemLocation(); + IndexedSeq values = + fs.ls().map(LocationDescription$.MODULE$::locationToPublicUrl).toIndexedSeq(); int size = values.size(); String[] result = new String[size]; diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringFromNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringFromNode.java index f236ddbbf..109ac4b40 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringFromNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringFromNode.java @@ -95,4 +95,10 @@ protected String fromTimestamp(TimestampObject value) { protected String fromInterval(IntervalObject value) { return value.toString(); } + + @Specialization + @TruffleBoundary + protected String fromLocation(LocationObject value) { + return value.getPublicDescription(); + } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadLinesNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadLinesNode.java index 19129155a..8daacf9c6 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadLinesNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadLinesNode.java @@ -13,7 +13,6 @@ package raw.runtime.truffle.ast.expressions.builtin.string_package; import com.oracle.truffle.api.dsl.*; -import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.NodeInfo; import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.runtime.generator.collection.StaticInitializers; @@ -21,7 +20,6 @@ import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.utils.TruffleCharInputStream; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "String.ReadLines") @NodeChild("location") @@ -30,12 +28,8 @@ public abstract class StringReadLinesNode extends ExpressionNode { @Specialization - static Object doExecute( - LocationObject locationObject, - String encoding, - @Bind("$node") Node thisNode, - @Cached(value = "getSourceContext(thisNode)", neverDefault = true) SourceContext context) { - TruffleInputStream stream = new TruffleInputStream(locationObject, context); + static Object doExecute(LocationObject locationObject, String encoding) { + TruffleInputStream stream = new TruffleInputStream(locationObject); TruffleCharInputStream charStream = new TruffleCharInputStream(stream, encoding); return new ReadLinesCollection(charStream); } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadNode.java index fae666c5b..53477f84e 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/builtin/string_package/StringReadNode.java @@ -14,7 +14,6 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.*; -import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.NodeInfo; import java.io.IOException; import java.io.Reader; @@ -25,7 +24,6 @@ import raw.runtime.truffle.runtime.primitives.ErrorObject; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "String.Read") @NodeChild("location") @@ -34,12 +32,8 @@ public abstract class StringReadNode extends ExpressionNode { @Specialization @TruffleBoundary - protected static Object doExecute( - LocationObject locationObject, - String encoding, - @Bind("$node") Node thisNode, - @Cached(value = "getSourceContext(thisNode)", neverDefault = true) SourceContext context) { - TruffleInputStream stream = new TruffleInputStream(locationObject, context); + protected static Object doExecute(LocationObject locationObject, String encoding) { + TruffleInputStream stream = new TruffleInputStream(locationObject); try { Reader reader = stream.getReader(encoding); try { diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/IntNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/IntNode.java index 6b53479b9..05448be2c 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/IntNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/IntNode.java @@ -12,30 +12,28 @@ package raw.runtime.truffle.ast.expressions.literals; -import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.VirtualFrame; import raw.runtime.truffle.ExpressionNode; public final class IntNode extends ExpressionNode { - private final String value; + private final int value; - public IntNode(String value) { + public IntNode(int value) { this.value = value; } + public IntNode(String value) { + this.value = Integer.parseInt(value); + } + @Override public int executeInt(VirtualFrame virtualFrame) { - return parseInt(); + return value; } @Override public Object executeGeneric(VirtualFrame virtualFrame) { - return parseInt(); - } - - @TruffleBoundary - public int parseInt() { - return Integer.parseInt(value); + return value; } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/LocationConstNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/LocationConstNode.java new file mode 100644 index 000000000..f7eb0e8f0 --- /dev/null +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/expressions/literals/LocationConstNode.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.runtime.truffle.ast.expressions.literals; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import raw.compiler.rql2.api.LocationDescription$; +import raw.runtime.truffle.ExpressionNode; +import raw.runtime.truffle.RawContext; +import raw.runtime.truffle.runtime.primitives.LocationObject; +import raw.sources.api.Location; +import raw.utils.RawSettings; + +public class LocationConstNode extends ExpressionNode { + + private final byte[] value; + private final String publicDescription; + + public LocationConstNode(byte[] value, String publicDescription) { + this.value = value; + this.publicDescription = publicDescription; + } + + @Override + public final Object executeGeneric(VirtualFrame virtualFrame) { + RawSettings rawSettings = RawContext.get(this).getSettings(); + Location location = getLocation(rawSettings); + return new LocationObject(location, publicDescription); + } + + @CompilerDirectives.TruffleBoundary + public Location getLocation(RawSettings rawSettings) { + return LocationDescription$.MODULE$.toLocation( + LocationDescription$.MODULE$.deserialize(value), rawSettings); + } +} diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQuery.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQuery.java index 09a55ba0c..fc86704d9 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQuery.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/jdbc/JdbcQuery.java @@ -18,36 +18,32 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import raw.client.api.LocationDescription; import raw.runtime.truffle.runtime.exceptions.rdbms.JdbcExceptionHandler; import raw.runtime.truffle.runtime.exceptions.rdbms.JdbcReaderRawTruffleException; -import raw.runtime.truffle.runtime.primitives.DateObject; -import raw.runtime.truffle.runtime.primitives.DecimalObject; -import raw.runtime.truffle.runtime.primitives.TimeObject; -import raw.runtime.truffle.runtime.primitives.TimestampObject; -import raw.sources.api.SourceContext; -import raw.sources.jdbc.api.JdbcLocationProvider; +import raw.runtime.truffle.runtime.primitives.*; import raw.utils.RawException; +import raw.utils.RawSettings; public class JdbcQuery { + private final String publicDescription; private final Connection connection; private final ResultSet rs; private final JdbcExceptionHandler exceptionHandler; - private final String url; @TruffleBoundary public JdbcQuery( - LocationDescription locationDescription, + LocationObject locationObject, String query, - SourceContext context, + RawSettings rawSettings, JdbcExceptionHandler exceptionHandler) { + this.publicDescription = locationObject.getPublicDescription(); this.exceptionHandler = exceptionHandler; - this.url = locationDescription.url(); try { - connection = JdbcLocationProvider.build(locationDescription, context).getJdbcConnection(); PreparedStatement stmt; - int fetchSize = context.settings().getInt("raw.runtime.rdbms.fetch-size"); + int fetchSize = rawSettings.getInt("raw.runtime.rdbms.fetch-size"); + // FIXME (msb): What is crashes during construction? Must release connection. + connection = locationObject.getJdbcServerLocation().getJdbcConnection(); try { stmt = connection.prepareStatement(query); stmt.setFetchSize(fetchSize); @@ -56,7 +52,7 @@ public JdbcQuery( throw exceptionHandler.rewrite(e, this); } } catch (RawException e) { - // exceptions due to location errors (e.g. connection failures) are turned into runtime + // Exceptions due to location errors (e.g. connection failures) are turned into runtime // exceptions. throw new JdbcReaderRawTruffleException(e.getMessage(), this, e, null); } @@ -212,7 +208,7 @@ boolean isNull(String colName, Node node) { } } - public String location() { - return url; + public String getPublicDescription() { + return publicDescription; } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/json/reader/JsonReadValueNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/json/reader/JsonReadValueNode.java index bc5e1d7c3..2ce938a4f 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/json/reader/JsonReadValueNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/json/reader/JsonReadValueNode.java @@ -18,14 +18,12 @@ import com.oracle.truffle.api.nodes.DirectCallNode; import com.oracle.truffle.api.nodes.NodeInfo; import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.RawContext; import raw.runtime.truffle.ast.io.json.reader.JsonParserNodes.CloseJsonParserNode; import raw.runtime.truffle.ast.io.json.reader.JsonParserNodes.InitJsonParserNode; import raw.runtime.truffle.ast.io.json.reader.JsonParserNodes.NextTokenJsonParserNode; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.utils.TruffleCharInputStream; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "Json.ReadArray") public class JsonReadValueNode extends ExpressionNode { @@ -47,8 +45,6 @@ public class JsonReadValueNode extends ExpressionNode { private NextTokenJsonParserNode nextTokenNode = JsonParserNodesFactory.NextTokenJsonParserNodeGen.create(); - private final SourceContext sourceContext = RawContext.get(this).getSourceContext(); - private JsonParser parser; public JsonReadValueNode( @@ -64,7 +60,7 @@ public Object executeGeneric(VirtualFrame virtualFrame) { LocationObject locationObject = (LocationObject) locationExp.executeGeneric(virtualFrame); String encoding = (String) encodingExp.executeGeneric(virtualFrame); - TruffleInputStream truffleInputStream = new TruffleInputStream(locationObject, sourceContext); + TruffleInputStream truffleInputStream = new TruffleInputStream(locationObject); TruffleCharInputStream stream = new TruffleCharInputStream(truffleInputStream, encoding); parser = initParserNode.execute(this, stream); diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlParseCollectionNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlParseCollectionNode.java index 726c430bd..826ec3514 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlParseCollectionNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlParseCollectionNode.java @@ -16,10 +16,8 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.NodeInfo; import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.RawContext; import raw.runtime.truffle.ast.ProgramExpressionNode; import raw.runtime.truffle.runtime.iterable.sources.XmlParseCollection; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "XmlParseCollection") public class XmlParseCollectionNode extends ExpressionNode { @@ -51,7 +49,6 @@ public Object executeGeneric(VirtualFrame virtualFrame) { String datetimeFormat = (String) datetimeFormatExp.executeGeneric(virtualFrame); RawTruffleXmlParserSettings settings = new RawTruffleXmlParserSettings(dateFormat, timeFormat, datetimeFormat); - SourceContext context = RawContext.get(this).getSourceContext(); return new XmlParseCollection(text, parseNextRootCallTarget, settings); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlReadValueNode.java b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlReadValueNode.java index 46f91fc06..70fc261c5 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlReadValueNode.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/ast/io/xml/parser/XmlReadValueNode.java @@ -17,13 +17,11 @@ import com.oracle.truffle.api.nodes.DirectCallNode; import com.oracle.truffle.api.nodes.NodeInfo; import raw.runtime.truffle.ExpressionNode; -import raw.runtime.truffle.RawContext; import raw.runtime.truffle.runtime.exceptions.xml.XmlParserRawTruffleException; import raw.runtime.truffle.runtime.exceptions.xml.XmlReaderRawTruffleException; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.runtime.truffle.utils.TruffleCharInputStream; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; @NodeInfo(shortName = "XmlReadValue") public class XmlReadValueNode extends ExpressionNode { @@ -37,8 +35,6 @@ public class XmlReadValueNode extends ExpressionNode { @Child private DirectCallNode childDirectCall; - private final SourceContext sourceContext = RawContext.get(this).getSourceContext(); - public XmlReadValueNode( ExpressionNode locationExp, ExpressionNode encodingExp, @@ -60,8 +56,7 @@ public Object executeGeneric(VirtualFrame virtualFrame) { try { LocationObject locationObject = (LocationObject) locationExp.executeGeneric(virtualFrame); String encoding = (String) encodingExp.executeGeneric(virtualFrame); - TruffleInputStream truffleInputStream = - new TruffleInputStream(locationObject, this.sourceContext); + TruffleInputStream truffleInputStream = new TruffleInputStream(locationObject); TruffleCharInputStream stream = new TruffleCharInputStream(truffleInputStream, encoding); String dateFormat = (String) dateFormatExp.executeGeneric(virtualFrame); String timeFormat = (String) timeFormatExp.executeGeneric(virtualFrame); diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/exceptions/rdbms/JdbcReaderRawTruffleException.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/exceptions/rdbms/JdbcReaderRawTruffleException.java index 0cdd60498..4ab817cdb 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/exceptions/rdbms/JdbcReaderRawTruffleException.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/exceptions/rdbms/JdbcReaderRawTruffleException.java @@ -22,6 +22,8 @@ public class JdbcReaderRawTruffleException extends RawTruffleRuntimeException { @CompilerDirectives.TruffleBoundary public JdbcReaderRawTruffleException(String message, JdbcQuery rs, Throwable e, Node location) { super( - String.format("failed to read from database %s: %s", rs.location(), message), e, location); + String.format("failed to read from database %s: %s", rs.getPublicDescription(), message), + e, + location); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/StaticInitializers.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/StaticInitializers.java index 142ee25f4..c8a4e7e95 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/StaticInitializers.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/StaticInitializers.java @@ -25,7 +25,7 @@ import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.group_by.OffHeapGroupByKey; import raw.runtime.truffle.runtime.generator.collection.off_heap_generator.off_heap.order_by.OffHeapGroupByKeys; import raw.runtime.truffle.utils.IOUtils; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; public class StaticInitializers { @@ -37,9 +37,9 @@ public static void kryoWriteInt(Output kryoOutput, int size) { @CompilerDirectives.TruffleBoundary public static FileOutputStream getGroupByKeyNewDiskBuffer( OffHeapGroupByKey offHeapGroupByKey, Node node) { - SourceContext sourceContext = RawContext.get(node).getSourceContext(); + RawSettings settings = RawContext.get(node).getSettings(); File file; - file = IOUtils.getScratchFile("groupby.", ".kryo", sourceContext).toFile(); + file = IOUtils.getScratchFile("groupby.", ".kryo", settings).toFile(); offHeapGroupByKey.getSpilledBuffers().add(file); try { return new FileOutputStream(file); @@ -52,8 +52,8 @@ public static FileOutputStream getGroupByKeyNewDiskBuffer( public static FileOutputStream groupByKeysNextFile( OffHeapGroupByKeys offHeapGroupByKeys, Node node) { File file; - SourceContext sourceContext = RawContext.get(node).getSourceContext(); - file = IOUtils.getScratchFile("orderby.", ".kryo", sourceContext).toFile(); + RawSettings settings = RawContext.get(node).getSettings(); + file = IOUtils.getScratchFile("orderby.", ".kryo", settings).toFile(); offHeapGroupByKeys.getSpilledBuffers().add(file); try { return new FileOutputStream(file); @@ -65,8 +65,8 @@ public static FileOutputStream groupByKeysNextFile( @CompilerDirectives.TruffleBoundary public static FileOutputStream distinctNextFile(OffHeapDistinct offHeapDistinct, Node node) { File file; - SourceContext sourceContext = RawContext.get(node).getSourceContext(); - file = IOUtils.getScratchFile("distinct.", ".kryo", sourceContext).toFile(); + RawSettings settings = RawContext.get(node).getSettings(); + file = IOUtils.getScratchFile("distinct.", ".kryo", settings).toFile(); offHeapDistinct.getSpilledBuffers().add(file); try { return new FileOutputStream(file); @@ -82,27 +82,22 @@ public static void kryoOutputClose(Output kryoOutput) { public static int getKryoOutputBufferSize(Node node) { return (int) - RawContext.get(node) - .getSourceContext() - .settings() - .getMemorySize("raw.runtime.kryo.output-buffer-size"); + RawContext.get(node).getSettings().getMemorySize("raw.runtime.kryo.output-buffer-size"); } @CompilerDirectives.TruffleBoundary public static long[] getContextValues(Node node) { - SourceContext sourceContext = RawContext.get(node).getSourceContext(); + RawSettings rawSettings = RawContext.get(node).getSettings(); long[] contextValues = new long[3]; - contextValues[0] = - sourceContext.settings().getMemorySize("raw.runtime.external.disk-block-max-size"); + contextValues[0] = rawSettings.getMemorySize("raw.runtime.external.disk-block-max-size"); contextValues[1] = getKryoOutputBufferSize(node); - contextValues[2] = - (int) sourceContext.settings().getMemorySize("raw.runtime.kryo.input-buffer-size"); + contextValues[2] = (int) rawSettings.getMemorySize("raw.runtime.kryo.input-buffer-size"); return contextValues; } @CompilerDirectives.TruffleBoundary - public static SourceContext getSourceContext(Node node) { - return RawContext.get(node).getSourceContext(); + public static RawSettings getRawSettings(Node node) { + return RawContext.get(node).getSettings(); } @CompilerDirectives.TruffleBoundary diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/ComputeNextNodes.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/ComputeNextNodes.java index 5d102dd43..916c5dd0b 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/ComputeNextNodes.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/ComputeNextNodes.java @@ -28,6 +28,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import raw.runtime.truffle.RawContext; import raw.runtime.truffle.RawLanguage; import raw.runtime.truffle.ast.io.csv.reader.CsvParserNodes; import raw.runtime.truffle.ast.io.json.reader.JsonParserNodes; @@ -59,7 +60,7 @@ import raw.runtime.truffle.utils.RawTruffleStringCharStream; import raw.runtime.truffle.utils.TruffleCharInputStream; import raw.runtime.truffle.utils.TruffleInputStream; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; public class ComputeNextNodes { @@ -506,12 +507,9 @@ static void init( CsvReadComputeNext computeNext, @Bind("$node") Node thisNode, @Cached @Cached.Shared("initCsv") CsvParserNodes.InitCsvParserNode initParser, - @Cached @Cached.Shared("closeCsv") CsvParserNodes.CloseCsvParserNode closeParser, - @Cached(value = "getSourceContext(thisNode)", allowUncached = true, neverDefault = true) - SourceContext sourceContext) { + @Cached @Cached.Shared("closeCsv") CsvParserNodes.CloseCsvParserNode closeParser) { try { - TruffleInputStream truffleInputStream = - new TruffleInputStream(computeNext.getLocation(), sourceContext); + TruffleInputStream truffleInputStream = new TruffleInputStream(computeNext.getLocation()); computeNext.setStream( new TruffleCharInputStream(truffleInputStream, computeNext.getEncoding())); computeNext.setParser( @@ -561,12 +559,10 @@ static void init( @Bind("$node") Node thisNode, @Cached JsonParserNodes.InitJsonParserNode initParser, @Cached JsonParserNodes.CloseJsonParserNode closeParser, - @Cached JsonParserNodes.NextTokenJsonParserNode nextToken, - @Cached(value = "getSourceContext(thisNode)", allowUncached = true, neverDefault = true) - SourceContext sourceContext) { + @Cached JsonParserNodes.NextTokenJsonParserNode nextToken) { try { TruffleInputStream truffleInputStream = - new TruffleInputStream(computeNext.getLocationObject(), sourceContext); + new TruffleInputStream(computeNext.getLocationObject()); computeNext.setStream( new TruffleCharInputStream(truffleInputStream, computeNext.getEncoding())); computeNext.setParser(initParser.execute(thisNode, computeNext.getStream())); @@ -616,15 +612,10 @@ static void init(Node node, XmlParseComputeNext computeNext) { } @Specialization - static void init( - Node node, - XmlReadComputeNext computeNext, - @Bind("$node") Node thisNode, - @Cached(value = "getSourceContext(thisNode)", allowUncached = true, neverDefault = true) - SourceContext sourceContext) { + static void init(Node node, XmlReadComputeNext computeNext) { try { TruffleInputStream truffleInputStream = - new TruffleInputStream(computeNext.getLocationObject(), sourceContext); + new TruffleInputStream(computeNext.getLocationObject()); computeNext.setStream( new TruffleCharInputStream(truffleInputStream, computeNext.getEncoding())); computeNext.setParser( @@ -812,12 +803,10 @@ static void init( LoopNode loopNode, @Cached @Cached.Shared("getGenerator") IterableNodes.GetGeneratorNode getGeneratorNode, @Cached(inline = false) @Cached.Shared("init") GeneratorNodes.GeneratorInitNode initNode, - @Cached @Cached.Shared("close1") GeneratorNodes.GeneratorCloseNode closeNode, - @Cached(value = "getSourceContext(thisNode)", allowUncached = true) - SourceContext sourceContext) { + @Cached @Cached.Shared("close1") GeneratorNodes.GeneratorCloseNode closeNode) { - computeNext.setDiskRight( - IOUtils.getScratchFile("cartesian.", ".kryo", sourceContext).toFile()); + RawSettings settings = RawContext.get(thisNode).getSettings(); + computeNext.setDiskRight(IOUtils.getScratchFile("cartesian.", ".kryo", settings).toFile()); // save right to disk Object rightGen = getGeneratorNode.execute(thisNode, computeNext.getRightIterable()); diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/sources/JdbcQueryComputeNext.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/sources/JdbcQueryComputeNext.java index 1bb94a187..6925e3b20 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/sources/JdbcQueryComputeNext.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/generator/collection/abstract_generator/compute_next/sources/JdbcQueryComputeNext.java @@ -16,14 +16,14 @@ import raw.runtime.truffle.ast.io.jdbc.JdbcQuery; import raw.runtime.truffle.runtime.exceptions.rdbms.JdbcExceptionHandler; import raw.runtime.truffle.runtime.primitives.LocationObject; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; public class JdbcQueryComputeNext { private final LocationObject dbLocation; private final String query; + private final RawSettings rawSettings; private final RootCallTarget rowParserCallTarget; - private final SourceContext context; private final JdbcExceptionHandler exceptionHandler; private JdbcQuery rs = null; @@ -31,12 +31,12 @@ public class JdbcQueryComputeNext { public JdbcQueryComputeNext( LocationObject dbLocation, String query, - SourceContext context, + RawSettings rawSettings, RootCallTarget rowParserCallTarget, JdbcExceptionHandler exceptionHandler) { - this.context = context; this.dbLocation = dbLocation; this.query = query; + this.rawSettings = rawSettings; this.rowParserCallTarget = rowParserCallTarget; this.exceptionHandler = exceptionHandler; } @@ -46,12 +46,7 @@ public JdbcQuery getRs() { } public void init() { - this.rs = - new JdbcQuery( - this.dbLocation.getLocationDescription(), - this.query, - this.context, - this.exceptionHandler); + this.rs = new JdbcQuery(this.dbLocation, this.query, this.rawSettings, this.exceptionHandler); } public void close() { diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/iterable/sources/JdbcQueryCollection.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/iterable/sources/JdbcQueryCollection.java index 8e17ffa12..2ed2488c8 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/iterable/sources/JdbcQueryCollection.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/iterable/sources/JdbcQueryCollection.java @@ -25,34 +25,34 @@ import raw.runtime.truffle.runtime.generator.collection.abstract_generator.AbstractGenerator; import raw.runtime.truffle.runtime.generator.collection.abstract_generator.compute_next.sources.JdbcQueryComputeNext; import raw.runtime.truffle.runtime.primitives.LocationObject; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; @ExportLibrary(InteropLibrary.class) public class JdbcQueryCollection implements TruffleObject { private final LocationObject dbLocation; private final String query; + private final RawSettings rawSettings; private final RootCallTarget rowParserCallTarget; - private final SourceContext context; private final JdbcExceptionHandler exceptionHandler; public JdbcQueryCollection( LocationObject dbLocation, String query, - SourceContext context, + RawSettings rawSettings, RootCallTarget rowParserCallTarget, JdbcExceptionHandler exceptionHandler) { this.dbLocation = dbLocation; this.query = query; + this.rawSettings = rawSettings; this.rowParserCallTarget = rowParserCallTarget; - this.context = context; this.exceptionHandler = exceptionHandler; } public AbstractGenerator getGenerator() { return new AbstractGenerator( new JdbcQueryComputeNext( - dbLocation, query, context, rowParserCallTarget, exceptionHandler)); + dbLocation, query, rawSettings, rowParserCallTarget, exceptionHandler)); } // InteropLibrary: Iterable diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/primitives/LocationObject.java b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/primitives/LocationObject.java index 12a7cafd7..0aa12a31f 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/primitives/LocationObject.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/runtime/primitives/LocationObject.java @@ -14,38 +14,85 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidBufferOffsetException; import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.ExportLibrary; import com.oracle.truffle.api.library.ExportMessage; -import raw.runtime.truffle.runtime.exceptions.RawTruffleInternalErrorException; -import raw.runtime.truffle.runtime.list.IntList; -import raw.runtime.truffle.runtime.list.StringList; -import raw.client.api.*; -import scala.collection.JavaConverters; -import scala.collection.immutable.HashMap; -import scala.collection.immutable.Map; +import java.nio.ByteOrder; +import raw.compiler.rql2.api.LocationDescription; +import raw.compiler.rql2.api.LocationDescription$; +import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; +import raw.sources.api.Location; +import raw.sources.bytestream.api.ByteStreamLocation; +import raw.sources.bytestream.http.HttpByteStreamLocation; +import raw.sources.filesystem.api.FileSystemLocation; +import raw.sources.jdbc.api.JdbcServerLocation; +/** + * Truffle object representing a location. + * + *

The location is represented in Interop as a serialized byte array. The public description is + * represented in Interop as a string. + */ @ExportLibrary(InteropLibrary.class) public final class LocationObject implements TruffleObject { + private final Location location; + private final String publicDescription; private final LocationDescription locationDescription; + private final byte[] byteArray; @TruffleBoundary - public LocationObject(String url) { - this.locationDescription = new LocationDescription(url, new HashMap<>()); + public LocationObject(Location location, String publicDescription) { + this.location = location; + this.publicDescription = publicDescription; + this.locationDescription = LocationDescription$.MODULE$.toLocationDescription(location); + this.byteArray = LocationDescription$.MODULE$.serialize(this.locationDescription); } - @TruffleBoundary - public LocationObject(String url, Map params) { - this.locationDescription = new LocationDescription(url, params); + public Location getLocation() { + return location; } @TruffleBoundary - public LocationObject(LocationDescription locationDescription) { - this.locationDescription = locationDescription; + public String getPublicDescription() { + return publicDescription; + } + + public ByteStreamLocation getByteStreamLocation() { + if (location instanceof ByteStreamLocation) { + return (ByteStreamLocation) location; + } else { + throw new RawTruffleRuntimeException("not a byte stream location"); + } + } + + public FileSystemLocation getFileSystemLocation() { + if (location instanceof FileSystemLocation) { + return (FileSystemLocation) location; + } else { + throw new RawTruffleRuntimeException("not a file system location"); + } + } + + public JdbcServerLocation getJdbcServerLocation() { + if (location instanceof JdbcServerLocation) { + return (JdbcServerLocation) location; + } else { + throw new RawTruffleRuntimeException("not a database location"); + } } - public LocationDescription getLocationDescription() { - return locationDescription; + public HttpByteStreamLocation getHttpByteStreamLocation() { + if (location instanceof HttpByteStreamLocation) { + return (HttpByteStreamLocation) location; + } else { + throw new RawTruffleRuntimeException("not an HTTP location"); + } + } + + public byte[] getBytes() { + return byteArray; } @ExportMessage @@ -55,56 +102,75 @@ final boolean isString() { @ExportMessage final String asString() { - return locationDescription.url(); + return publicDescription; + } + + @ExportMessage + boolean hasArrayElements() { + return true; + } + + @ExportMessage + final long getArraySize() { + return byteArray.length; } @ExportMessage - final boolean hasMembers() { + final byte readArrayElement(long index) throws ArrayIndexOutOfBoundsException { + return byteArray[(int) index]; + } + + @ExportMessage + final boolean isArrayElementReadable(long index) { + return index >= 0 && index < byteArray.length; + } + + @ExportMessage + final boolean hasBufferElements() { return true; } @ExportMessage - final Object getMembers(boolean includeInternal) { - String[] keys = - JavaConverters.asJavaCollection(locationDescription.settings().keys()).stream() - .map(LocationSettingKey::key) - .toArray(String[]::new); - return new StringList(keys); + final long getBufferSize() { + return byteArray.length; + } + + @ExportMessage + final byte readBufferByte(long byteOffset) throws InvalidBufferOffsetException { + int idx = (int) byteOffset; + if (idx < 0 || idx >= byteArray.length) { + throw InvalidBufferOffsetException.create(idx, 1); + } + return byteArray[idx]; + } + + @ExportMessage + final short readBufferShort(ByteOrder order, long byteOffset) + throws UnsupportedMessageException, InvalidBufferOffsetException { + throw new UnsupportedOperationException(); + } + + @ExportMessage + final int readBufferInt(ByteOrder order, long byteOffset) + throws UnsupportedMessageException, InvalidBufferOffsetException { + throw new UnsupportedOperationException(); + } + + @ExportMessage + final long readBufferLong(ByteOrder order, long byteOffset) + throws UnsupportedMessageException, InvalidBufferOffsetException { + throw new UnsupportedOperationException(); } @ExportMessage - final boolean isMemberReadable(String member) { - return locationDescription.settings().keySet().contains(new LocationSettingKey(member)); + final float readBufferFloat(ByteOrder order, long byteOffset) + throws UnsupportedMessageException, InvalidBufferOffsetException { + throw new UnsupportedOperationException(); } @ExportMessage - final Object readMember(String member) { - return switch (locationDescription.settings().get(new LocationSettingKey(member)).get()) { - case LocationIntSetting v -> v.value(); - case LocationStringSetting v -> v.value(); - case LocationBinarySetting v -> { - int size = v.value().size(); - byte[] bytes = new byte[size]; - for (int i = 0; i < size; i++) { - bytes[i] = (byte) v.value().apply(i); - } - yield new BinaryObject(bytes); - } - case LocationBooleanSetting v -> v.value(); - case LocationDurationSetting v -> v.value(); - case LocationKVSetting v -> { - // a hash - int size = v.map().size(); - String[][] pairsArray = new String[size][]; - for (int i = 0; i < size; i++) { - String key = v.map().apply(i)._1(); - String value = v.map().apply(i)._2(); - pairsArray[i] = new String[] {key, value}; - } - yield new LocationKVSettingHash(pairsArray); - } - case LocationIntArraySetting v -> new IntList(v.value()); - default -> throw new RawTruffleInternalErrorException(); - }; + final double readBufferDouble(ByteOrder order, long byteOffset) + throws UnsupportedMessageException, InvalidBufferOffsetException { + throw new UnsupportedOperationException(); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/IOUtils.java b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/IOUtils.java index c9ba2c8c4..7b73e34a5 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/IOUtils.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/IOUtils.java @@ -18,13 +18,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; -import raw.sources.api.SourceContext; +import raw.utils.RawSettings; public class IOUtils { @TruffleBoundary - public static Path getScratchPath(SourceContext context) { - Path p = Paths.get(context.settings().getString("raw.runtime.scratch-path", true)); + public static Path getScratchPath(RawSettings rawSettings) { + Path p = Paths.get(rawSettings.getString("raw.runtime.scratch-path", true)); if (!Files.exists(p)) { try { Files.createDirectories(p); @@ -36,9 +36,9 @@ public static Path getScratchPath(SourceContext context) { } @TruffleBoundary - public static Path getScratchFile(String prefix, String suffix, SourceContext context) { + public static Path getScratchFile(String prefix, String suffix, RawSettings rawSettings) { try { - return Files.createTempFile(getScratchPath(context), prefix, suffix); + return Files.createTempFile(getScratchPath(rawSettings), prefix, suffix); } catch (IOException ex) { throw new RawTruffleRuntimeException("failed to create scratch file", ex); } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleCharInputStream.java b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleCharInputStream.java index 379a480a0..6d21f23b9 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleCharInputStream.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleCharInputStream.java @@ -32,6 +32,6 @@ public Reader getReader() { @TruffleBoundary public String positionDescription() { - return "url: " + stream.getUrl(); + return "location: " + stream.getPublicDescription(); } } diff --git a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleInputStream.java b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleInputStream.java index 52cbc4a00..326a1add0 100644 --- a/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleInputStream.java +++ b/snapi-truffle/src/main/java/raw/runtime/truffle/utils/TruffleInputStream.java @@ -18,31 +18,26 @@ import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; import raw.runtime.truffle.runtime.primitives.LocationObject; import raw.sources.api.Encoding; -import raw.sources.api.SourceContext; import raw.sources.bytestream.api.ByteStreamLocation; -import raw.sources.bytestream.api.ByteStreamLocationProvider; import raw.utils.RawException; import scala.util.Either; public class TruffleInputStream { private final LocationObject locationObject; - private final SourceContext context; - - public TruffleInputStream(LocationObject locationObject, SourceContext context) { - this.context = context; + public TruffleInputStream(LocationObject locationObject) { this.locationObject = locationObject; } @TruffleBoundary - public String getUrl() { - return locationObject.getLocationDescription().url(); + public String getPublicDescription() { + return locationObject.getPublicDescription(); } @TruffleBoundary public ByteStreamLocation getLocation() { try { - return ByteStreamLocationProvider.build(locationObject.getLocationDescription(), context); + return locationObject.getByteStreamLocation(); } catch (RawException ex) { throw new RawTruffleRuntimeException(ex.getMessage(), ex, null); } diff --git a/snapi-truffle/src/main/resources/META-INF/services/raw.compiler.rql2.api.EntryExtension b/snapi-truffle/src/main/resources/META-INF/services/raw.compiler.rql2.api.EntryExtension index 8454424e2..8c970ba56 100644 --- a/snapi-truffle/src/main/resources/META-INF/services/raw.compiler.rql2.api.EntryExtension +++ b/snapi-truffle/src/main/resources/META-INF/services/raw.compiler.rql2.api.EntryExtension @@ -93,7 +93,7 @@ raw.compiler.snapi.truffle.builtin.list_extension.TruffleFromListEntry raw.compiler.snapi.truffle.builtin.list_extension.TruffleUnsafeFromListEntry raw.compiler.snapi.truffle.builtin.list_extension.TruffleGroupListEntry raw.compiler.snapi.truffle.builtin.list_extension.TruffleExistsListEntry -raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationBuildEntry +raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationFromStringEntry raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationDescribeEntry raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLsEntry raw.compiler.snapi.truffle.builtin.location_extension.TruffleLocationLlEntry diff --git a/sources/src/main/java/module-info.java b/sources/src/main/java/module-info.java index 4c76eab13..8b0737a17 100644 --- a/sources/src/main/java/module-info.java +++ b/sources/src/main/java/module-info.java @@ -57,21 +57,15 @@ requires com.microsoft.sqlserver.jdbc; requires mysql.connector.j; requires ojdbc10; + requires snowflake.jdbc; + requires dropbox.core.sdk; requires raw.utils; - requires raw.client; - exports raw.auth.api; - exports raw.rest.client; - exports raw.rest.common; - exports raw.creds.api; - exports raw.creds.client; - exports raw.creds.local; - exports raw.creds.protocol; exports raw.sources.api; exports raw.sources.bytestream.api; exports raw.sources.bytestream.github; exports raw.sources.bytestream.http; - exports raw.sources.bytestream.in_memory; + exports raw.sources.bytestream.inmemory; exports raw.sources.filesystem.api; exports raw.sources.filesystem.dropbox; exports raw.sources.filesystem.local; @@ -85,11 +79,4 @@ exports raw.sources.jdbc.sqlserver; exports raw.sources.jdbc.oracle; exports raw.sources.jdbc.teradata; - - opens raw.auth.api to - com.fasterxml.jackson.databind; - opens raw.rest.common to - com.fasterxml.jackson.databind; - opens raw.creds.api to - com.fasterxml.jackson.databind; } diff --git a/sources/src/main/resources/reference.conf b/sources/src/main/resources/reference.conf index 8653b35c1..e7439822c 100644 --- a/sources/src/main/resources/reference.conf +++ b/sources/src/main/resources/reference.conf @@ -17,27 +17,29 @@ raw.sources { network-timeout = 300 s login-timeout = 30 s } -} -raw.sources.dropbox.clientId = "" -raw.sources.bytestream.http { - connect-timeout = 20 s - read-timeout = 120 s - conn-pool-max-per-route = 4096 - conn-pool-max-total = 32768 -} -raw.sources.s3 { - connect-timeout = 60 s - read-timeout = 120 s - max-connections = 50 + bytestream { + http { + connect-timeout = 20 s + read-timeout = 120 s + } + } + dropbox { + clientId = "" + } + s3 { + connect-timeout = 60 s + read-timeout = 120 s + max-connections = 50 - # Hadoop s3a filesystem will make a nested loop of retries with the 2 next settings. - # so if max-retries = 7 and max-attempts = 3, it will make 7*3 = 21 retries - # see fs.s3a.attempts.maximum and fs.s3a.retry.limit - max-retries = 10 - max-attempts = 0 - # Initial delay between s3a retries, see fs.s3a.retry.interval - retry-interval = 100 ms - tmp-dir = ${java.io.tmpdir}/s3 + # Hadoop s3a filesystem will make a nested loop of retries with the 2 next settings. + # so if max-retries = 7 and max-attempts = 3, it will make 7*3 = 21 retries + # see fs.s3a.attempts.maximum and fs.s3a.retry.limit + max-retries = 10 + max-attempts = 0 + # Initial delay between s3a retries, see fs.s3a.retry.interval + retry-interval = 100 ms + tmp-dir = ${java.io.tmpdir}/s3 - default-region = eu-west-1 -} + default-region = eu-west-1 + } +} \ No newline at end of file diff --git a/sources/src/main/scala/raw/auth/api/AuthException.scala b/sources/src/main/scala/raw/auth/api/AuthException.scala deleted file mode 100644 index 59b08e1c4..000000000 --- a/sources/src/main/scala/raw/auth/api/AuthException.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.auth.api - -import raw.utils.RawServiceException - -abstract class AuthException(message: String, cause: Throwable = null) - extends RawServiceException(s"authentication error: $message", cause) - -// Status 400 -class GenericAuthException(message: String, cause: Throwable = null) extends AuthException(message, cause) - -// Status 401. -class UnauthorizedException(message: String, cause: Throwable = null) extends AuthException(message, cause) - -// Status 403. -class ForbiddenException(message: String, cause: Throwable = null) extends AuthException(message, cause) diff --git a/sources/src/main/scala/raw/auth/api/Tokens.scala b/sources/src/main/scala/raw/auth/api/Tokens.scala deleted file mode 100644 index a6e8e79ce..000000000 --- a/sources/src/main/scala/raw/auth/api/Tokens.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.auth.api - -import java.time.Instant -import com.fasterxml.jackson.annotation.JsonIgnore - -trait Token { - def accessToken: String - - def remainingValidityInSeconds: Long -} - -final case class TokenWithValidity(accessToken: String, validUntil: Instant) { - @JsonIgnore - def isExpired: Boolean = { - Instant.now().isAfter(validUntil) - } -} - -trait TokenProvider { - def token: TokenWithValidity -} diff --git a/sources/src/main/scala/raw/creds/api/Credentials.scala b/sources/src/main/scala/raw/creds/api/Credentials.scala deleted file mode 100644 index 87ace99c4..000000000 --- a/sources/src/main/scala/raw/creds/api/Credentials.scala +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.api - -import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType} -import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} -import raw.utils.Uid - -import java.time.Instant -import java.util.Locale - -sealed trait Credential - -/** - * Dropbox - */ -final case class DropboxToken(accessToken: String, tokenType: String, uid: Uid) extends Credential - -/** - * S3. - * - * It may be necessary to specify a region explicitly. - * If no region is given, the Java AWS SDK will try to resolve the region as described here: - * https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/java-dg-region-selection.html - * If the region is not given explicitly and is configured externally (env variable, aws config file, - * EC2 instance metadata) then the SDK will not be able to resolve the region of the bucket. - * - * TODO (msb): Rename to credentials to maybeAwsCredentials - */ -final case class S3Bucket(name: String, region: Option[String], credentials: Option[AWSCredentials]) extends Credential - -final case class AWSCredentials(accessKey: String, secretKey: String) - -/** - * HTTP. - */ - -final case class HttpCredential(prefixUrl: String, credentials: HttpAuth) extends Credential - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[OauthClientCredentials], name = "oauth-client"), - new JsonType(value = classOf[OauthToken], name = "oauth-token"), - new JsonType(value = classOf[BasicAuthCredentials], name = "basic-auth") - ) -) -sealed trait HttpAuth - -final case class OauthClientCredentials(clientId: String, clientSecret: String, tokenUrl: String, useBasicAuth: Boolean) - extends HttpAuth - -final case class OauthToken(token: String, refreshToken: Option[String], tokenUrl: Option[String]) extends HttpAuth - -final case class BasicAuthCredentials(user: String, password: String) extends HttpAuth - -final case class AccessToken(accessToken: String) extends HttpAuth - -/** - * Relational Database. - */ - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[PostgresqlCredential], name = "postgresql"), - new JsonType(value = classOf[MySqlCredential], name = "mysql"), - new JsonType(value = classOf[OracleCredential], name = "oracle"), - new JsonType(value = classOf[SqlServerCredential], name = "sqlserver"), - new JsonType(value = classOf[TeradataCredential], name = "teradata"), - new JsonType(value = classOf[SnowflakeCredential], name = "snowflake") - ) -) -sealed trait RelationalDatabaseCredential extends Credential { - def host: String - def port: Option[Int] - def username: Option[String] - def password: Option[String] -} - -final case class PostgresqlCredential( - host: String, - port: Option[Int], - database: String, - username: Option[String], - password: Option[String], - schema: Option[String] = None -) extends RelationalDatabaseCredential - -final case class MySqlCredential( - host: String, - port: Option[Int], - database: String, - username: Option[String], - password: Option[String] -) extends RelationalDatabaseCredential - -final case class OracleCredential( - host: String, - port: Option[Int], - database: String, - username: Option[String], - password: Option[String], - schema: Option[String] = None -) extends RelationalDatabaseCredential - -final case class SqlServerCredential( - host: String, - port: Option[Int], - database: String, - username: Option[String], - password: Option[String], - schema: Option[String] = None -) extends RelationalDatabaseCredential - -// list of possible parameters (might include the port also) -//https://teradata-docs.s3.amazonaws.com/doc/connectivity/jdbc/reference/current/jdbcug_chapter_2.html -final case class TeradataCredential( - host: String, - port: Option[Int], - username: Option[String], - password: Option[String], - parameters: Map[String, String] = Map.empty, - schema: Option[String] = None -) extends RelationalDatabaseCredential - -final case class SnowflakeCredential( - accountIdentifier: String, - database: String, - username: Option[String], - password: Option[String], - parameters: Map[String, String] = Map.empty, - schema: Option[String] = None -) extends RelationalDatabaseCredential { - val host = s"$accountIdentifier.snowflakecomputing.com" - val port = None -} - -case class ExternalConnectorCredentialId(name: String, connectorType: AbstractConnectorType) - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "repr") -@JsonSubTypes( - Array( - new JsonType(value = classOf[SalesforceConnectorType], name = "SALESFORCE"), - new JsonType(value = classOf[JiraConnectorType], name = "JIRA"), - new JsonType(value = classOf[GithubConnectorType], name = "GITHUB") - ) -) -trait AbstractConnectorType { - def repr: String -} -case class SalesforceConnectorType() extends AbstractConnectorType { - override def repr: String = "SALESFORCE" -} -case class JiraConnectorType() extends AbstractConnectorType { - override def repr: String = "JIRA" -} -case class GithubConnectorType() extends AbstractConnectorType { - override def repr: String = "GITHUB" -} - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "connectorType") -@JsonSubTypes( - Array( - new JsonType(value = classOf[ExternalConnectorSalesforceCredential], name = "SALESFORCE"), - new JsonType(value = classOf[ExternalConnectorJiraCredential], name = "JIRA"), - new JsonType(value = classOf[ExternalConnectorGithubCredential], name = "GITHUB") - ) -) -sealed trait ExternalConnectorCredential extends Credential { - def connectorType: AbstractConnectorType - def sensitiveFields: List[String] -} - -final case class ExternalConnectorSalesforceCredential( - url: String, - username: String, - password: String, - securityToken: String, - clientId: String, - apiVersion: String, - customObjects: Seq[String] -) extends ExternalConnectorCredential { - override def connectorType: AbstractConnectorType = SalesforceConnectorType() - override def sensitiveFields: List[String] = List("password", "securityToken") -} - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[StandardAccessToken], name = "DEFAULT"), - new JsonType(value = classOf[PersonalAccessToken], name = "PAT") - ) -) -sealed trait TokenType -case class StandardAccessToken() extends TokenType -case class PersonalAccessToken() extends TokenType - -final case class ExternalConnectorJiraCredential( - baseUrl: String, - username: String, - token: String, - tokenType: TokenType -) extends ExternalConnectorCredential { - override def connectorType: AbstractConnectorType = JiraConnectorType() - override def sensitiveFields: List[String] = List("token") -} - -final case class ExternalConnectorGithubCredential( - baseUrl: String, - token: String -) extends ExternalConnectorCredential { - override def connectorType: AbstractConnectorType = GithubConnectorType() - override def sensitiveFields: List[String] = List("token") -} - -final case class Secret(name: String, value: String) extends Credential - -// Http credentials -object HttpCredentialType extends Enumeration { - type CredentialType = Value - - val ClientCredentials, Token, UserPass = Value - - def apply(name: String): Value = { - values - .find(_.toString.toLowerCase(Locale.ROOT) == name.toLowerCase(Locale.ROOT)) - .getOrElse(throw new IllegalArgumentException(s"Invalid credential type: $name")) - } -} - -/** - * Represents an Http credential, a way of obtaining an access token. This can be a client credential, an access token with - * a refresh token, a plain access token or any other way of generating an access token. - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[BasicAuthCredential], name = "basic-auth"), - new JsonType(value = classOf[TokenCredential], name = "oauth2-token"), - new JsonType(value = classOf[ClientCredentialsCredential], name = "oauth2-client-credentials") - ) -) -sealed trait NewHttpCredential extends Credential { - def options: Map[String, String] -} - -final case class BasicAuthCredential( - username: String, - password: String, - options: Map[String, String] -) extends NewHttpCredential - -final case class TokenCredential( - provider: OAuth2Provider.Value, - accessToken: String, - expiresBy: Option[Instant], - scopes: Option[Seq[String]], - refreshToken: Option[String], - options: Map[String, String] -) extends NewHttpCredential - -final case class ClientCredentialsCredential( - provider: OAuth2Provider.Value, - clientId: String, - clientSecret: String, - options: Map[String, String], - maybeAccessToken: Option[String] = None, - maybeScopes: Option[Seq[String]] = None, - maybeExpiresBy: Option[Instant] = None -) extends NewHttpCredential - -case class HttpCredentialId(name: String, credType: HttpCredentialType.Value) - -// For internal Http clients, represents the required information to open an Http connection from a query. -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[BearerToken], name = "bearer-token"), - new JsonType(value = classOf[BasicAuth], name = "basic-auth"), - new JsonType(value = classOf[CustomHeaderToken], name = "custom-header") - ) -) -sealed trait NewHttpAuth -final case class BearerToken(token: String, options: Map[String, String]) extends NewHttpAuth -final case class BasicAuth(username: String, password: String, options: Map[String, String]) extends NewHttpAuth -final case class CustomHeaderToken(header: String, token: String, options: Map[String, String]) extends NewHttpAuth diff --git a/sources/src/main/scala/raw/creds/api/CredentialsService.scala b/sources/src/main/scala/raw/creds/api/CredentialsService.scala deleted file mode 100644 index 5b3b151b6..000000000 --- a/sources/src/main/scala/raw/creds/api/CredentialsService.scala +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.api - -import raw.utils.{AuthenticatedUser, RawService} -import raw.creds.api.CredentialsService.{bucketNameRegex, credNamePattern} - -object CredentialsService { - // Refer to https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html - private val bucketNameRegex = """^[a-z0-9][a-z0-9.-]*[a-z0-9]$""".r - - // A single word (letters, numbers or underscore) or a word , followed by zero or more words or dashes or dots, followed by a word. - // examples of valid names: "a", "a1", "a-b" - // examples of invalid names: "a b", "a-", "-a" - // Used name validation for secrets, new-http-credentials and rdbms-servers. - // things like s3 buckets, http-prefixes have their own validation - private val credNamePattern = "^(\\w|\\w[\\w\\.-]*\\w)$".r - - //Should we have a list of regions? - // maybe in the future amazon will add more and we have to keep updating - private val validRegions = Array( - "us-east-2", - "us-east-1", - "us-west-1", - "us-west-2", - "ap-east-1", - "ap-south-1", - "ap-northeast-3", - "ap-northeast-2", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-north-1", - "me-south-1", - "sa-east-1", - "us-gov-east-1", - "us-gov-west-1" - ) - - def validateBucketRegion(region: String): Boolean = { - validRegions.contains(region) - } -} - -trait CredentialsService extends RawService { - - /** S3 buckets */ - - private def validateS3BucketName(name: String): Unit = { - if (name == null) { - throw new CredentialsException("null name") - } else if (name.length < 3) { - throw new CredentialsException(s"credential name too small: '$name'") - } else if (name.length > 63) { - throw new CredentialsException(s"credential name too long: '$name'") - } else if (bucketNameRegex.findFirstMatchIn(name).isEmpty) { - throw new CredentialsException(s"credential name is invalid: '$name'") - } - } - - final def registerS3Bucket(user: AuthenticatedUser, bucket: S3Bucket): Boolean = { - validateS3BucketName(bucket.name) - doRegisterS3Bucket(user, bucket) - } - - protected def doRegisterS3Bucket(user: AuthenticatedUser, bucket: S3Bucket): Boolean - - def getS3Bucket(user: AuthenticatedUser, name: String): Option[S3Bucket] - - def existsS3Bucket(user: AuthenticatedUser, name: String): Boolean = getS3Bucket(user, name).isDefined - - def listS3Buckets(user: AuthenticatedUser): List[String] - - def unregisterS3Bucket(user: AuthenticatedUser, name: String): Boolean - - /** Dropbox tokens */ - - def registerDropboxToken(user: AuthenticatedUser, dropboxToken: DropboxToken): Boolean - - def getDropboxToken(user: AuthenticatedUser): Option[DropboxToken] - - def existsDropboxToken(user: AuthenticatedUser): Boolean = getDropboxToken(user).isDefined - - def unregisterDropboxToken(user: AuthenticatedUser): Boolean - - /** ExternalConnector */ - - def registerExternalConnectorCredential( - user: AuthenticatedUser, - name: String, - credential: ExternalConnectorCredential - ): Boolean - - def getExternalConnectorCredential(user: AuthenticatedUser, name: String): Option[ExternalConnectorCredential] - - def existsExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean = - getExternalConnectorCredential(user, name).isDefined - - def listExternalConnectorCredentials(user: AuthenticatedUser): List[ExternalConnectorCredentialId] - - def unregisterExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean - - /** Http Credentials */ - - protected def validateNewHttpCredentialName(name: String) = { - if (name == null) { - throw new CredentialsException("null name") - } else if (name == "") { - throw new CredentialsException(s"credential name is empty") - } else if (name.length > 64) { - throw new CredentialsException(s"credential name too long: '$name'") - } else if (credNamePattern.findFirstMatchIn(name).isEmpty) { - throw new CredentialsException(s"credential name is invalid: '$name'") - } - } - - // TODO: Distinguish between an OAuth2Credential from a plain access token. OAuth2Credential is a way of obtaining a token - // and can be using refresh tokens, client id/secret or a static access token - protected def doRegisterNewHttpCredential(user: AuthenticatedUser, name: String, token: NewHttpCredential): Boolean - - final def registerNewHttpCredential(user: AuthenticatedUser, name: String, token: NewHttpCredential): Boolean = { - validateNewHttpCredentialName(name) - doRegisterNewHttpCredential(user, name, token) - } - - def getNewHttpCredential(user: AuthenticatedUser, name: String): Option[NewHttpCredential] - - def existsNewHttpCredential(user: AuthenticatedUser, name: String): Boolean = - getNewHttpCredential(user, name).isDefined - - def unregisterNewHttpCredential(user: AuthenticatedUser, name: String): Boolean - - def listNewHttpCredentials(user: AuthenticatedUser): List[HttpCredentialId] - - /** - * Obtain an access token from the OAuth2 credential with the given name. The service will try do the necessary to - * generate a valid token, using the OAuth2 flow associated with the credential (client credentials, refresh token). - * This may not always be possible, if the credential does not have enough information to obtain a new token. In this - * case, it may return expired access tokens. - */ - def getNewHttpAuth(user: AuthenticatedUser, name: String): Option[NewHttpAuth] - - /** RDBMS servers */ - - protected def validateRDBMSName(name: String) = { - if (name == null) { - throw new CredentialsException("null name") - } else if (name == "") { - throw new CredentialsException(s"credential name is empty") - } else if (name.length > 64) { - throw new CredentialsException(s"credential name too long: '$name'") - } else if (credNamePattern.findFirstMatchIn(name).isEmpty) { - throw new CredentialsException(s"credential name is invalid: '$name'") - } - } - - final def registerRDBMSServer(user: AuthenticatedUser, name: String, db: RelationalDatabaseCredential): Boolean = { - validateRDBMSName(name) - doRegisterRDBMSServer(user, name, db) - } - - protected def doRegisterRDBMSServer(user: AuthenticatedUser, name: String, db: RelationalDatabaseCredential): Boolean - - def getRDBMSServer(user: AuthenticatedUser, name: String): Option[RelationalDatabaseCredential] - - def existsRDBMSServer(user: AuthenticatedUser, name: String): Boolean = getRDBMSServer(user, name).isDefined - - def listRDBMSServers(user: AuthenticatedUser): List[String] - - def unregisterRDBMSServer(user: AuthenticatedUser, name: String): Boolean - - /** HTTP credentials */ - - protected def validateHttpPrefix(prefix: String) = { - if (prefix == null) throw new CredentialsException("null prefix") - else if (prefix == "") throw new CredentialsException("empty prefix") - } - - @deprecated("Use registerNewHttpCredentials instead", since = "2022-06-14") - final def registerHTTPCred(user: AuthenticatedUser, cred: HttpCredential): Boolean = { - validateHttpPrefix(cred.prefixUrl) - doRegisterHTTPCred(user, cred) - } - - protected def doRegisterHTTPCred(user: AuthenticatedUser, cred: HttpCredential): Boolean - - @deprecated("Use getNewHttpCredentials instead", since = "2022-06-14") - def getHTTPCred(user: AuthenticatedUser, url: String): Option[HttpCredential] - - @deprecated("Use existsNewHttpCredentials instead", since = "2022-06-14") - def existsHTTPCred(user: AuthenticatedUser, url: String): Boolean = getHTTPCred(user, url).isDefined - - @deprecated("Use listNewHttpCredentials instead", since = "2022-06-14") - def listHTTPCreds(user: AuthenticatedUser): List[String] - - @deprecated("Use unregisterNewHttpCredentials instead", since = "2022-06-14") - def unregisterHTTPCred(user: AuthenticatedUser, url: String): Boolean - - protected def doRegisterSecret(user: AuthenticatedUser, secret: Secret): Boolean - - private def validateSecretName(name: String) = { - if (name == null) { - throw new CredentialsException("null name") - } else if (name == "") { - throw new CredentialsException(s"credential name is empty") - } else if (name.length > 64) { - throw new CredentialsException(s"credential name too long: '$name'") - } else if (credNamePattern.findFirstMatchIn(name).isEmpty) { - throw new CredentialsException(s"credential name is invalid: '$name'") - } - } - - final def registerSecret(user: AuthenticatedUser, secret: Secret): Boolean = { - validateSecretName(secret.name) - doRegisterSecret(user, secret) - } - - def getSecret(user: AuthenticatedUser, name: String): Option[Secret] - - def existsSecret(user: AuthenticatedUser, name: String): Boolean = getSecret(user, name).isDefined - - def listSecrets(user: AuthenticatedUser): List[String] - - def unregisterSecret(user: AuthenticatedUser, name: String): Boolean - - /** - * Returns a PostgreSQL FDW database for the given user, creating the database - * if it does not already exist. - * - * @param user the AuthenticatedUser which corresponds to the combination of RAW Organization and Repository - * @return the database name that corresponds to the given user. - */ - def getUserDb(user: AuthenticatedUser): String - - /** - * Returns a JDBC connection string for the specified user, creating the corresponding database - * if it does not already exist. - * - * @param user the AuthenticatedUser we need the JDBC connection string for - * @return the JDBC connection string for the specified user - */ - def getUserJdbc(user: AuthenticatedUser): String -} diff --git a/sources/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala b/sources/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala deleted file mode 100644 index a13dd2a9e..000000000 --- a/sources/src/main/scala/raw/creds/api/CredentialsServiceProvider.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.api - -import raw.creds.client.ClientCredentialsService -import raw.creds.local.LocalCredentialsService -import raw.utils.RawSettings - -object CredentialsServiceProvider { - - private val CREDS_IMPL = "raw.creds.impl" - - def apply()(implicit settings: RawSettings): CredentialsService = { - settings.getStringOpt(CREDS_IMPL) match { - case Some("client") => new ClientCredentialsService - case Some("local") => new LocalCredentialsService - case Some(impl) => throw new CredentialsException(s"cannot find credentials service: $impl") - case None => throw new CredentialsException("no credentials service defined") - } - } - -} diff --git a/sources/src/main/scala/raw/creds/api/OAuth2Provider.scala b/sources/src/main/scala/raw/creds/api/OAuth2Provider.scala deleted file mode 100644 index 3157d07ef..000000000 --- a/sources/src/main/scala/raw/creds/api/OAuth2Provider.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.api - -import java.util.Locale - -object OAuth2Provider extends Enumeration { - type OAuth2Provider = Value - - val Dropbox, Auth0, Shopify, Zoho, LinkedIn, Twitter, GoogleApi, Generic = Value - - def apply(name: String): Value = { - val sanitizedName = name.toLowerCase(Locale.ROOT).replace("-", "") - values - .find(_.toString.toLowerCase(Locale.ROOT) == sanitizedName) - .getOrElse(throw new CredentialsException(s"invalid credential type: $name")) - } -} diff --git a/sources/src/main/scala/raw/creds/client/ClientCredentials.scala b/sources/src/main/scala/raw/creds/client/ClientCredentials.scala deleted file mode 100644 index 6e1c4a791..000000000 --- a/sources/src/main/scala/raw/creds/client/ClientCredentials.scala +++ /dev/null @@ -1,383 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.client - -import com.typesafe.scalalogging.StrictLogging -import org.apache.hc.core5.http.HttpStatus -import raw.utils.{AuthenticatedUser, RawSettings} -import raw.creds.api._ -import raw.creds.protocol._ -import raw.rest.client.{ClientAPIException, RestClient} - -import java.net.URI - -class ClientCredentials(serverAddress: URI)(implicit settings: RawSettings) extends StrictLogging { - - private val restClient = new RestClient(serverAddress, None, "credentials") - - /** S3 buckets */ - - def registerS3Bucket(user: AuthenticatedUser, bucket: S3Bucket): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/s3/register", - RegisterS3BucketCredential(user, bucket), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "s3CredentialsAlreadyExists" => false - } - } - - def getS3Bucket(user: AuthenticatedUser, name: String): Option[S3Bucket] = { - try { - Some(restClient.doJsonPost[S3Bucket]("2/s3/get", GetS3BucketCredential(user, name), withAuth = false)) - } catch { - case ex: ClientAPIException if ex.errorCode == "s3CredentialsNotFound" => None - } - } - - def existsS3Bucket(user: AuthenticatedUser, name: String): Boolean = getS3Bucket(user, name).isDefined - - def listS3Buckets(user: AuthenticatedUser): List[String] = { - restClient.doJsonPost[List[String]]("2/s3/list", ListS3BucketCredentials(user), withAuth = false) - } - - def unregisterS3Bucket(user: AuthenticatedUser, name: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/s3/unregister", - UnregisterS3BucketCredential(user, name), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "s3CredentialsNotFound" => false - } - } - - /** Dropbox tokens */ - - def registerDropboxToken(user: AuthenticatedUser, dropboxToken: DropboxToken): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/dropbox/register", - RegisterDropboxCredential(user, dropboxToken), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "dropboxTokenAlreadyExists" => false - } - } - - def getDropboxToken(user: AuthenticatedUser): Option[DropboxToken] = { - try { - Some(restClient.doJsonPost[DropboxToken]("2/dropbox/get", GetDropboxCredential(user), withAuth = false)) - } catch { - case ex: ClientAPIException if ex.errorCode == "dropboxTokenNotFound" => None - } - } - - def existsDropboxToken(user: AuthenticatedUser): Boolean = getDropboxToken(user).isDefined - - def unregisterDropboxToken(user: AuthenticatedUser): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/dropbox/unregister", - UnregisterDropboxCredential(user), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "dropboxTokenNotFound" => false - } - } - - /** Salesforce */ - - def registerExternalConnectorCredential( - user: AuthenticatedUser, - name: String, - externalConnectorCredential: ExternalConnectorCredential - ): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/connector/register", - RegisterExternalConnectorCredential(user, name, externalConnectorCredential), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "externalConnectorCredentialAlreadyExists" => false - } - } - - def getExternalConnectorCredential(user: AuthenticatedUser, name: String): Option[ExternalConnectorCredential] = { - try { - Some( - restClient - .doJsonPost[ExternalConnectorCredential]( - "2/connector/get", - GetExternalConnectorCredential(user, name), - withAuth = false - ) - ) - } catch { - case ex: ClientAPIException if ex.errorCode == "externalConnectorCredentialNotFound" => None - } - } - - def existsExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean = - getExternalConnectorCredential(user, name).isDefined - - def listExternalConnectorCredentials(user: AuthenticatedUser): List[ExternalConnectorCredentialId] = { - restClient.doJsonPost[List[ExternalConnectorCredentialId]]( - "2/connector/list", - ListExternalConnectorCredential(user), - withAuth = false - ) - } - - def unregisterExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/connector/unregister", - UnregisterExternalConnectorCredential(user, name), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "salesforceCredentialNotFound" => false - } - } - - /** HTTP Credentials */ - - def listHttpCredentials(user: AuthenticatedUser): List[HttpCredentialId] = { - restClient.doJsonPost[List[HttpCredentialId]]("2/http/list", ListNewHttpCredentials(user), withAuth = false) - } - - def registerHttpCredential(user: AuthenticatedUser, name: String, token: NewHttpCredential): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/http/register", - RegisterNewHttpCredential(user, name, token), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsAlreadyExists" => false - } - } - - def getHttpCredential(user: AuthenticatedUser, name: String): Option[NewHttpCredential] = { - try { - Some(restClient.doJsonPost[NewHttpCredential]("2/http/get", GetNewHttpCredential(user, name), withAuth = false)) - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsNotFound" => None - } - } - - def existsHttpCredential(user: AuthenticatedUser, name: String): Boolean = getHttpCredential(user, name).isDefined - - def unregisterHttpCredential(user: AuthenticatedUser, name: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/http/unregister", - UnregisterNewHttpCredential(user, name), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsNotFound" => false - } - } - - def getHttpAuth(user: AuthenticatedUser, name: String): Option[NewHttpAuth] = { - try { - Some( - restClient.doJsonPost[NewHttpAuth]("2/http/get-token", GetTokenNewHttpCredential(user, name), withAuth = false) - ) - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsNotFound" => None - } - } - - /** RDBMS servers */ - - def registerRDBMSServer(user: AuthenticatedUser, name: String, db: RelationalDatabaseCredential): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/rdbms/register", - RegisterRelationalDatabaseCredential(user, name, db), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "databaseCredentialsAlreadyExists" => false - } - } - - def getRDBMSServer(user: AuthenticatedUser, name: String): Option[RelationalDatabaseCredential] = { - try { - Some( - restClient.doJsonPost[RelationalDatabaseCredential]( - "2/rdbms/get", - GetRelationalDatabaseCredential(user, name), - withAuth = false - ) - ) - } catch { - case ex: ClientAPIException if ex.errorCode == "databaseCredentialsNotFound" => None - } - } - - def existsRDBMSServer(user: AuthenticatedUser, name: String): Boolean = getRDBMSServer(user, name).isDefined - - def listRDBMSServers(user: AuthenticatedUser): List[String] = { - restClient.doJsonPost[List[String]]("2/rdbms/list", ListRelationalDatabaseCredentials(user), withAuth = false) - } - - def unregisterRDBMSServer(user: AuthenticatedUser, name: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/rdbms/unregister", - UnregisterRelationalDatabaseCredential(user, name), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "databaseCredentialsNotFound" => false - - } - } - - /** HTTP credentials */ - - @deprecated("Use registerHttpCredential instead", since = "2022-06-14") - def registerHTTPCred(user: AuthenticatedUser, auth: HttpCredential): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/http-legacy/register", - RegisterHttpCredential(user, auth), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsAlreadyExists" => false - } - } - - @deprecated("Use getHttpCredential instead", since = "2022-06-14") - def getHTTPCred(user: AuthenticatedUser, url: String): Option[HttpCredential] = { - try { - Some(restClient.doJsonPost[HttpCredential]("2/http-legacy/get", GetHttpCredential(user, url), withAuth = false)) - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsNotFound" => None - } - } - - @deprecated("Use existsHttpCredential instead", since = "2022-06-14") - def existsHTTPCred(user: AuthenticatedUser, url: String): Boolean = getHTTPCred(user, url).isDefined - - @deprecated("Use listHttpCredentials instead", since = "2022-06-14") - def listHTTPCreds(user: AuthenticatedUser): List[String] = { - restClient.doJsonPost[List[String]]("2/http-legacy/list", ListHttpCredentials(user), withAuth = false) - } - - @deprecated("Use unregisterHttpCredential instead", since = "2022-06-14") - def unregisterHTTPCred(user: AuthenticatedUser, url: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/http-legacy/unregister", - UnregisterHttpCredential(user, url), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "httpCredentialsNotFound" => false - } - } - - /** Secrets */ - - def registerSecret(user: AuthenticatedUser, secret: Secret): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/secrets/register", - RegisterSecretCredential(user, secret), - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "secretAlreadyExists" => false - } - } - - def getSecret(user: AuthenticatedUser, name: String): Option[Secret] = { - try { - Some(restClient.doJsonPost[Secret]("2/secrets/get", GetSecretCredential(user, name), withAuth = false)) - } catch { - case ex: ClientAPIException if ex.errorCode == "secretNotFound" => None - } - } - - def existsSecret(user: AuthenticatedUser, name: String): Boolean = getSecret(user, name).isDefined - - def unregisterSecret(user: AuthenticatedUser, name: String): Boolean = { - try { - restClient.doJsonPostWithEmptyResponse( - "2/secrets/unregister", - UnregisterSecretCredential(user, name), - HttpStatus.SC_NO_CONTENT, - withAuth = false - ) - true - } catch { - case ex: ClientAPIException if ex.errorCode == "secretNotFound" => false - } - } - - def listSecrets(user: AuthenticatedUser): List[String] = { - restClient.doJsonPost[List[String]]("2/secrets/list", ListSecretCredentials(user), withAuth = false) - } - - def getUserDb(user: AuthenticatedUser): String = { - restClient.doJsonPost[String]( - "2/fdw/provision", - ProvisionFdwDbCredentials(user), - withAuth = false - ) - } - - def getUserJdbc(user: AuthenticatedUser): String = { - restClient.doJsonPost[String]( - "2/fdw/user-jdbc", - ProvisionFdwDbCredentials(user), - withAuth = false - ) - } - - def close(): Unit = { - restClient.close() - } - -} diff --git a/sources/src/main/scala/raw/creds/client/ClientCredentialsException.scala b/sources/src/main/scala/raw/creds/client/ClientCredentialsException.scala deleted file mode 100644 index 54c7858d4..000000000 --- a/sources/src/main/scala/raw/creds/client/ClientCredentialsException.scala +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.client - -import raw.creds.api.CredentialsException - -class ClientCredentialsException(message: String, cause: Throwable = null) extends CredentialsException(message, cause) diff --git a/sources/src/main/scala/raw/creds/client/ClientCredentialsService.scala b/sources/src/main/scala/raw/creds/client/ClientCredentialsService.scala deleted file mode 100644 index ec89f433c..000000000 --- a/sources/src/main/scala/raw/creds/client/ClientCredentialsService.scala +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.client - -import raw.creds.api._ -import raw.creds.client.ClientCredentialsService.SERVER_ADDRESS -import raw.rest.client.APIException -import raw.utils.{AuthenticatedUser, RawSettings} - -import java.net.URI - -object ClientCredentialsService { - private val SERVER_ADDRESS = "raw.creds.client.server-address" -} - -class ClientCredentialsService(implicit settings: RawSettings) extends CredentialsService { - - private val serverAddress = new URI(settings.getString(SERVER_ADDRESS)) - - private val client = new ClientCredentials(serverAddress) - - /** S3 buckets */ - - override protected def doRegisterS3Bucket(user: AuthenticatedUser, bucket: S3Bucket): Boolean = { - try { - client.registerS3Bucket(user, bucket) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getS3Bucket(user: AuthenticatedUser, name: String): Option[S3Bucket] = { - try { - client.getS3Bucket(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listS3Buckets(user: AuthenticatedUser): List[String] = { - try { - client.listS3Buckets(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterS3Bucket(user: AuthenticatedUser, name: String): Boolean = { - try { - client.unregisterS3Bucket(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** Dropbox tokens */ - - override def registerDropboxToken(user: AuthenticatedUser, dropboxToken: DropboxToken): Boolean = { - try { - client.registerDropboxToken(user, dropboxToken) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getDropboxToken(user: AuthenticatedUser): Option[DropboxToken] = { - try { - client.getDropboxToken(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterDropboxToken(user: AuthenticatedUser): Boolean = { - try { - client.unregisterDropboxToken(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** ExternalConnector */ - - override def registerExternalConnectorCredential( - user: AuthenticatedUser, - name: String, - externalConnectorCredential: ExternalConnectorCredential - ): Boolean = { - try { - client.registerExternalConnectorCredential(user, name, externalConnectorCredential) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getExternalConnectorCredential( - user: AuthenticatedUser, - name: String - ): Option[ExternalConnectorCredential] = { - try { - client.getExternalConnectorCredential(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listExternalConnectorCredentials(user: AuthenticatedUser): List[ExternalConnectorCredentialId] = { - try { - client.listExternalConnectorCredentials(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean = { - try { - client.unregisterExternalConnectorCredential(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** Http Credentials */ - - override protected def doRegisterNewHttpCredential( - user: AuthenticatedUser, - name: String, - token: NewHttpCredential - ): Boolean = { - try { - client.registerHttpCredential(user, name, token) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getNewHttpCredential(user: AuthenticatedUser, name: String): Option[NewHttpCredential] = { - try { - client.getHttpCredential(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterNewHttpCredential(user: AuthenticatedUser, name: String): Boolean = { - try { - client.unregisterHttpCredential(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listNewHttpCredentials(user: AuthenticatedUser): List[HttpCredentialId] = { - try { - client.listHttpCredentials(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getNewHttpAuth(user: AuthenticatedUser, name: String): Option[NewHttpAuth] = { - try { - client.getHttpAuth(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** RDBMS servers */ - - override protected def doRegisterRDBMSServer( - user: AuthenticatedUser, - name: String, - db: RelationalDatabaseCredential - ): Boolean = { - try { - client.registerRDBMSServer(user, name, db) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getRDBMSServer(user: AuthenticatedUser, name: String): Option[RelationalDatabaseCredential] = { - try { - client.getRDBMSServer(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listRDBMSServers(user: AuthenticatedUser): List[String] = { - try { - client.listRDBMSServers(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterRDBMSServer(user: AuthenticatedUser, name: String): Boolean = { - try { - client.unregisterRDBMSServer(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** HTTP credentials */ - - override protected def doRegisterHTTPCred(user: AuthenticatedUser, cred: HttpCredential): Boolean = { - try { - client.registerHTTPCred(user, cred) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getHTTPCred(user: AuthenticatedUser, url: String): Option[HttpCredential] = { - try { - client.getHTTPCred(user, url) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listHTTPCreds(user: AuthenticatedUser): List[String] = { - try { - client.listHTTPCreds(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterHTTPCred(user: AuthenticatedUser, url: String): Boolean = { - try { - client.unregisterHTTPCred(user, url) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - /** Secret credentials */ - - override def doRegisterSecret(user: AuthenticatedUser, secret: Secret): Boolean = { - try { - client.registerSecret(user, secret) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getSecret(user: AuthenticatedUser, name: String): Option[Secret] = { - try { - client.getSecret(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def listSecrets(user: AuthenticatedUser): List[String] = { - try { - client.listSecrets(user) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def unregisterSecret(user: AuthenticatedUser, name: String): Boolean = { - try { - client.unregisterSecret(user, name) - } catch { - case ex: APIException => throw new ClientCredentialsException(ex.getMessage, ex) - } - } - - override def getUserDb(user: AuthenticatedUser): String = { - client.getUserDb(user) - } - - override def getUserJdbc(user: AuthenticatedUser): String = { - client.getUserJdbc(user) - } - - override def doStop(): Unit = { - client.close() - } - -} diff --git a/sources/src/main/scala/raw/creds/local/LocalCredentialsService.scala b/sources/src/main/scala/raw/creds/local/LocalCredentialsService.scala deleted file mode 100644 index f667deb8d..000000000 --- a/sources/src/main/scala/raw/creds/local/LocalCredentialsService.scala +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.local - -import raw.utils.AuthenticatedUser -import raw.creds.api._ - -class LocalCredentialsService extends CredentialsService { - - override protected def doRegisterS3Bucket(user: AuthenticatedUser, bucket: S3Bucket): Boolean = { - false - } - - override def getS3Bucket(user: AuthenticatedUser, name: String): Option[S3Bucket] = { - None - } - - override def listS3Buckets(user: AuthenticatedUser): List[String] = { - List.empty - } - - override def unregisterS3Bucket(user: AuthenticatedUser, name: String): Boolean = { - false - } - - override def registerDropboxToken(user: AuthenticatedUser, dropboxToken: DropboxToken): Boolean = { - false - } - - override def getDropboxToken(user: AuthenticatedUser): Option[DropboxToken] = { - None - } - - override def unregisterDropboxToken(user: AuthenticatedUser): Boolean = { - false - } - - override def registerExternalConnectorCredential( - user: AuthenticatedUser, - name: String, - credential: ExternalConnectorCredential - ): Boolean = { - false - } - - override def getExternalConnectorCredential( - user: AuthenticatedUser, - name: String - ): Option[ExternalConnectorCredential] = { - None - } - - override def listExternalConnectorCredentials(user: AuthenticatedUser): List[ExternalConnectorCredentialId] = { - List.empty - } - - override def unregisterExternalConnectorCredential(user: AuthenticatedUser, name: String): Boolean = { - false - } - - override protected def doRegisterNewHttpCredential( - user: AuthenticatedUser, - name: String, - token: NewHttpCredential - ): Boolean = { - false - } - - override def getNewHttpCredential(user: AuthenticatedUser, name: String): Option[NewHttpCredential] = { - None - } - - override def unregisterNewHttpCredential(user: AuthenticatedUser, name: String): Boolean = { - false - } - - override def listNewHttpCredentials(user: AuthenticatedUser): List[HttpCredentialId] = { - List.empty - } - - override def getNewHttpAuth(user: AuthenticatedUser, name: String): Option[NewHttpAuth] = { - None - } - - override protected def doRegisterRDBMSServer( - user: AuthenticatedUser, - name: String, - db: RelationalDatabaseCredential - ): Boolean = { - false - } - - override def getRDBMSServer(user: AuthenticatedUser, name: String): Option[RelationalDatabaseCredential] = { - None - } - - override def listRDBMSServers(user: AuthenticatedUser): List[String] = { - List.empty - } - - override def unregisterRDBMSServer(user: AuthenticatedUser, name: String): Boolean = { - false - } - - override protected def doRegisterHTTPCred(user: AuthenticatedUser, cred: HttpCredential): Boolean = { - false - } - - override def getHTTPCred(user: AuthenticatedUser, url: String): Option[HttpCredential] = { - None - } - - override def listHTTPCreds(user: AuthenticatedUser): List[String] = { - List.empty - } - - override def unregisterHTTPCred(user: AuthenticatedUser, url: String): Boolean = { - false - } - - override def doRegisterSecret(user: AuthenticatedUser, secret: Secret): Boolean = { - false - } - - override def getSecret(user: AuthenticatedUser, name: String): Option[Secret] = { - val envVariable = s"SNAPI_${name.toUpperCase}" - sys.env.get(envVariable).map(v => Secret(envVariable, v)) - } - - override def listSecrets(user: AuthenticatedUser): List[String] = { - sys.env.keys.filter(_.startsWith("SNAPI_")).map(_.substring(6)).toList - } - - override def unregisterSecret(user: AuthenticatedUser, name: String): Boolean = { - false - } - - override def getUserDb(user: AuthenticatedUser): String = { - ??? - } - - override def getUserJdbc(user: AuthenticatedUser): String = { - ??? - } - - override def doStop(): Unit = {} - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/api/OAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/api/OAuth2Client.scala deleted file mode 100644 index 51c8940cd..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/api/OAuth2Client.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.api - -import com.fasterxml.jackson.annotation.{JsonSetter, Nulls} -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} -import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} -import com.typesafe.scalalogging.StrictLogging -import raw.utils.{RawService, RawSettings} - -import java.net.URLEncoder -import java.net.http.{HttpClient, HttpRequest} -import java.nio.charset.StandardCharsets - -abstract class OAuth2Client(implicit settings: RawSettings) extends RawService with StrictLogging { - - protected val connectTimeout = settings.getDuration("raw.creds.oauth2.connect-timeout") - - protected val readTimeout = settings.getDuration("raw.creds.oauth2.read-timeout") - - protected val mapper = new ObjectMapper with ClassTagExtensions - mapper.registerModule(DefaultScalaModule) - mapper.setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)) - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - - protected val httpClient = HttpClient.newBuilder - .connectTimeout(connectTimeout) - .version(HttpClient.Version.HTTP_1_1) - .build - - protected def ofFormData(data: Map[String, String]): HttpRequest.BodyPublisher = { - val builder = new StringBuffer() - for ((key, value) <- data) { - if (builder.length() != 0) builder.append('&') - builder - .append(URLEncoder.encode(key, StandardCharsets.UTF_8)) - .append("=") - .append(URLEncoder.encode(value, StandardCharsets.UTF_8)) - } - HttpRequest.BodyPublishers.ofString(builder.toString()) - } - - override def doStop(): Unit = { - httpClient.close() - } - - // Custom header required in the access token. - def customHeader: Option[String] = None - - def supportsRefreshToken: Boolean = false - - def supportsClientCredentials: Boolean = false - - /** - * Executes the OAuth2 refresh token flow to obtain a new access token. Implementation is specific to the provider. - */ - def newAccessTokenFromRefreshToken(refreshToken: String, options: Map[String, String]): RenewedAccessToken = ??? - - /** - * Executes the OAuth2 client credentials flow to obtain a new access token. Implementation is specific to the provider. - */ - def newAccessTokenFromClientCredentials( - clientId: String, - clientSecret: String, - options: Map[String, String] - ): RenewedAccessToken = ??? - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/auth0/Auth0OAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/auth0/Auth0OAuth2Client.scala deleted file mode 100644 index 160b527c9..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/auth0/Auth0OAuth2Client.scala +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.auth0 - -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import org.apache.commons.lang3.StringUtils -import org.apache.hc.core5.http.HttpHeaders -import org.apache.hc.core5.net.URIBuilder -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.utils.RawSettings - -import java.net.http.HttpRequest.BodyPublishers -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.time.Instant - -object Auth0OAuth2Client { - // Required options that must be provided by the user - val AUDIENCE_KEY = "audience" - val BASE_URL_KEY = "base_url" - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class Auth0TokenResponse( - accessToken: String, - expiresIn: Int, - scope: String, - tokenType: String, - refreshToken: Option[String] - ) - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class Auth0ErrorResponse(error: String, errorDescription: String) -} - -class Auth0OAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - - import Auth0OAuth2Client._ - - logger.debug("Creating new Auth0 OAuth2 client") - - override def supportsRefreshToken: Boolean = false - - override def supportsClientCredentials: Boolean = true - - override def newAccessTokenFromClientCredentials( - clientId: String, - clientSecret: String, - options: Map[String, String] - ): RenewedAccessToken = { - logger.debug(s"newAccessTokenFromClientCredentials: $clientId, $clientSecret") - - val audience = - options.get(AUDIENCE_KEY).getOrElse(throw new CredentialsException(s"""Missing "$AUDIENCE_KEY" in options""")) - val apiBaseUrl = - options.get(BASE_URL_KEY).getOrElse(throw new CredentialsException(s"""Missing "$BASE_URL_KEY" in options""")) - - val requestBody = mapper.writeValueAsString( - Map( - "grant_type" -> "client_credentials", - "client_id" -> clientId, - "client_secret" -> clientSecret, - "audience" -> audience - ) - ) - val uri = new URIBuilder(apiBaseUrl).appendPath("/oauth/token").normalizeSyntax().build() - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .header(HttpHeaders.CONTENT_TYPE, "application/json") - .POST(BodyPublishers.ofString(requestBody)) - .build() - logger.debug(s"Executing Auth0 client credentials flow, HTTP request $request") - - val response = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - - logger.debug(s"Response from Auth0: $response") - - // https://auth0.com/docs/api/authentication?http#post-oauth-access_token - handleErrorResponse(response) - if (response.statusCode() == 200) { - val auth0Token = mapper.readValue[Auth0TokenResponse](response.body()) - RenewedAccessToken( - auth0Token.accessToken, - Instant.now().plusSeconds(auth0Token.expiresIn), - StringUtils.split(auth0Token.scope), - auth0Token.refreshToken - ) - } else { - throw new CredentialsException(s"Unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } - - private def handleErrorResponse(response: HttpResponse[String]): Unit = { - response.statusCode() match { - case 200 | 201 | 202 | 204 => // Ok - case 400 => - val body = mapper.readValue[Auth0ErrorResponse](response.body()) - throw new CredentialsException( - s"Error calling Auth0 REST API: (${response.statusCode()}) ${body.error}: ${body.errorDescription}" - ) - - case 401 | 403 => - val errorResponse = mapper.readValue[Auth0ErrorResponse](response.body()) - throw new CredentialsException( - s"Authorization error calling Auth0 REST API: (${response.statusCode()}) ${errorResponse.error}: ${errorResponse.errorDescription}" - ) - - case _ => - logger.warn(s"Unexpected error calling REST API: (${response.statusCode()}) ${response.body()}") - throw new CredentialsException( - s"Unexpected error calling REST API. Status code: ${response.statusCode()}" - ) - } - } - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/dropbox/DropboxOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/dropbox/DropboxOAuth2Client.scala deleted file mode 100644 index 213169dc2..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/dropbox/DropboxOAuth2Client.scala +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.dropbox - -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import org.apache.commons.lang3.StringUtils -import org.apache.hc.core5.http.HttpHeaders -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.utils.RawSettings - -import java.net.URI -import java.net.http.HttpRequest.BodyPublishers -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.time.Instant -import scala.compat.java8.OptionConverters.RichOptionalGeneric - -object DropboxOAuth2Client { - // TODO (msb): This could be given as an option to the credential and therefore removed from here! - private val DROPBOX_CLIENT_ID = "raw.sources.dropbox.clientId" - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class DropboxAuth2TokenResponse( - accessToken: String, - expiresIn: Int, - scopes: String, - refreshToken: Option[String] - ) - -} - -class DropboxOAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - - import DropboxOAuth2Client._ - - // https://www.dropbox.com/developers/documentation/http/documentation#oauth2-token - logger.debug("Creating new Dropbox OAuth2 client") - - // TODO (ns): This could be given as an option to the credential - private val clientId = settings.getString(DROPBOX_CLIENT_ID) - - private val tokenUri = new URI("https://api.dropbox.com/oauth2/token") - private val testAccessUri = new URI("https://api.dropboxapi.com/2/users/get_space_usage") - // https://www.dropbox.com/developers/documentation/http/documentation#oauth2-token - - override def supportsRefreshToken: Boolean = true - - override def supportsClientCredentials: Boolean = false - - override def newAccessTokenFromRefreshToken( - refreshToken: String, - options: Map[String, String] - ): RenewedAccessToken = { - // https://www.dropbox.com/developers/documentation/http/documentation#oauth2-token - val formData = Map( - "grant_type" -> "refresh_token", - "refresh_token" -> refreshToken, - "client_id" -> clientId - ) - // Should send the codeVerifier? - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(tokenUri) - .POST(ofFormData(formData)) - .build() - logger.debug(s"Sending request to ${request.uri()}") - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - logger.debug(s"Response status: ${response.statusCode()}") - - handleErrorResponse(response) - - if (response.statusCode() == 200) { - val newToken = mapper.readValue[DropboxAuth2TokenResponse](response.body()) - RenewedAccessToken( - newToken.accessToken, - Instant.now().plusSeconds(newToken.expiresIn), - StringUtils.split(newToken.scopes), - newToken.refreshToken.filterNot(_.isBlank) - ) - } else { - throw new CredentialsException(s"Unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } - - def testAccess(accessToken: String): Unit = { - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(testAccessUri) - .header(HttpHeaders.AUTHORIZATION, s"Bearer $accessToken") - .POST(BodyPublishers.noBody()) - .build() - - val response = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - handleErrorResponse(response) - } - - /** Generic error handling logic. */ - private def handleErrorResponse(response: HttpResponse[String]): Unit = { - response.statusCode() match { - case 200 | 201 | 202 | 204 => // Ok - case 400 => - logger.warn(s"Error contacting REST API: ${response.body()}") - response.headers().firstValue(HttpHeaders.CONTENT_TYPE).asScala match { - case Some("application/json") => - val body = mapper.readValue[Map[String, String]](response.body()) - throw new CredentialsException( - s"Could not exchange code: ${body("error")}: ${body("error_description")} }" - ) - case _ => throw new CredentialsException(s"Could not exchange code: ${response.body()}") - } - case 401 => - logger.warn(s"Not authorized calling REST API: (${response.statusCode()}) ${response.body()}") - throw new CredentialsException( - s"Not authorized calling REST API. Status code: ${response.statusCode()}" - ) - - case _ => - logger.warn(s"Error calling REST API: (${response.statusCode()}) ${response.body()}") - throw new CredentialsException( - s"Unexpected error calling REST API. Status code: ${response.statusCode()}" - ) - } - } - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/generic/GenericOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/generic/GenericOAuth2Client.scala deleted file mode 100644 index 5d7def7dd..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/generic/GenericOAuth2Client.scala +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2024 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.generic - -import raw.creds.oauth2.api.OAuth2Client -import raw.utils.RawSettings - -class GenericOAuth2Client(implicit settings: RawSettings) extends OAuth2Client { - // Just uses the default OAuth2Client implementation. -} diff --git a/sources/src/main/scala/raw/creds/oauth2/google/GoogleApiKeyOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/google/GoogleApiKeyOAuth2Client.scala deleted file mode 100644 index 2fc929a0c..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/google/GoogleApiKeyOAuth2Client.scala +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.google - -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import pdi.jwt.{Jwt, JwtAlgorithm, JwtClaim} -import org.apache.hc.core5.http.HttpHeaders -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.utils.RawSettings - -import java.io.IOException -import java.net.URI -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.security.KeyFactory -import java.security.spec.{InvalidKeySpecException, PKCS8EncodedKeySpec} -import java.time.Instant -import java.util.Base64 - -object GoogleApiKeyOAuth2Client { - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class GoogleAuth2TokenResponse( - accessToken: String, - expiresIn: Int, - tokenType: String, - refreshToken: Option[String] - ) - -} - -class GoogleApiKeyOAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - - import GoogleApiKeyOAuth2Client._ - - override def supportsRefreshToken: Boolean = false - - override def supportsClientCredentials: Boolean = true - - /** - * Executes the OAuth2 client credentials flow to obtain a new access token. Implementation is specific to the provider. - */ - override def newAccessTokenFromClientCredentials( - clientId: String, - clientSecret: String, - options: Map[String, String] - ): RenewedAccessToken = { - - def getOption(key: String) = - options.getOrElse(key, throw new CredentialsException(s"""missing "$key" in options""")) - - val privateKey = clientSecret - val clientEmail = getOption("client_email") - val tokenUri = getOption("token_uri") - val scope = getOption("scope") - - val uri = new URI(tokenUri) - try { - val cleaned = privateKey - .replace("-----BEGIN PRIVATE KEY-----", "") - .replace("-----END PRIVATE KEY-----", "") - .replaceAll("\\s+", "") - - val pkcs8EncodedBytes = Base64.getDecoder().decode(cleaned) - - val keySpec = new PKCS8EncodedKeySpec(pkcs8EncodedBytes) - val kf = KeyFactory.getInstance("RSA"); - val privKey = kf.generatePrivate(keySpec); - val currentTime = Instant.now.getEpochSecond - - val claim = JwtClaim() - .by(clientEmail) - .to(tokenUri) - .issuedAt(currentTime) - .expiresAt(currentTime + 3600) // 1 hour - .+("scope", scope) - - val jwtToken = Jwt.encode(claim, privKey, JwtAlgorithm.RS256) - - val formData = Map( - "grant_type" -> "urn:ietf:params:oauth:grant-type:jwt-bearer", - "assertion" -> jwtToken - ) - - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .header(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded") - .POST(ofFormData(formData)) - .build() - logger.debug(s"Sending request to ${request.uri()}") - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - logger.debug(s"Response status: ${response.statusCode()}") - - if (response.statusCode() == 200) { - val newToken = mapper.readValue[GoogleAuth2TokenResponse](response.body()) - - RenewedAccessToken( - newToken.accessToken, - Instant.now().plusSeconds(newToken.expiresIn), - Seq.empty, - newToken.refreshToken - ) - } else { - throw new CredentialsException(s"unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } catch { - case ex: JsonProcessingException => throw new CredentialsException( - s"error processing json response while getting token from $uri", - ex - ) - case ex: InvalidKeySpecException => - throw new CredentialsException(s"error trying to get google-api access token: invalid private key", ex) - case ex: IOException => throw new CredentialsException(s"error getting google api token from $uri", ex) - } - } - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/linkedin/LinkedInOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/linkedin/LinkedInOAuth2Client.scala deleted file mode 100644 index d3918f8c6..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/linkedin/LinkedInOAuth2Client.scala +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.linkedin - -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import org.apache.hc.core5.http.HttpHeaders -import org.apache.hc.core5.net.URIBuilder -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.utils.RawSettings - -import java.io.IOException -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.time.Instant - -object LinkedInOAuth2Client { - private val CLIENT_ID_KEY = "client_id" - private val CLIENT_SECRET_KEY = "client_secret" - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class LinkedInAuth2TokenResponse( - accessToken: String, - expiresIn: Int, - refreshToken: Option[String], - refreshTokenExpiresIn: Option[Int] - ) - -} - -class LinkedInOAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - - import LinkedInOAuth2Client._ - - override def supportsRefreshToken: Boolean = true - - override def supportsClientCredentials: Boolean = false - - /** - * Executes the OAuth2 refresh token flow to obtain a new access token. Implementation is specific to the provider. - */ - override def newAccessTokenFromRefreshToken( - refreshToken: String, - options: Map[String, String] - ): RenewedAccessToken = { - - // https://docs.microsoft.com/en-us/linkedin/shared/authentication/programmatic-refresh-tokens - val clientId = - options.getOrElse(CLIENT_ID_KEY, throw new CredentialsException(s"""missing "$CLIENT_ID_KEY" in options""")) - val clientSecret = options.getOrElse( - CLIENT_SECRET_KEY, - throw new CredentialsException(s"""missing "$CLIENT_SECRET_KEY" in options""") - ) - try { - val uri = new URIBuilder("https://www.linkedin.com/oauth/v2/accessToken").build() - - val formData = Map( - "grant_type" -> "refresh_token", - "refresh_token" -> refreshToken, - "client_id" -> clientId, - "client_secret" -> clientSecret - ) - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .header(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded") - .POST(ofFormData(formData)) - .build() - - logger.debug(s"Sending request to ${request.uri()}") - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - logger.debug(s"Response status: ${response.statusCode()}") - - if (response.statusCode() == 200) { - val newToken = mapper.readValue[LinkedInAuth2TokenResponse](response.body()) - - RenewedAccessToken( - newToken.accessToken, - Instant.now().plusSeconds(newToken.expiresIn), - Seq.empty, - newToken.refreshToken - ) - } else { - throw new CredentialsException(s"unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } catch { - case ex: JsonProcessingException => - throw new CredentialsException(s"error processing json response while refreshing LinkedIn token", ex) - case ex: IOException => throw new CredentialsException(s"error refreshing LinkedIn token", ex) - } - - } - -} diff --git a/sources/src/main/scala/raw/creds/oauth2/shopify/ShopifyOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/shopify/ShopifyOAuth2Client.scala deleted file mode 100644 index 8dd0b2032..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/shopify/ShopifyOAuth2Client.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2024 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.shopify - -import raw.creds.oauth2.api.OAuth2Client -import raw.utils.RawSettings - -class ShopifyOAuth2Client(implicit settings: RawSettings) extends OAuth2Client { - - override def customHeader: Option[String] = Some("X-Shopify-Access-Token") -} diff --git a/sources/src/main/scala/raw/creds/oauth2/twitter/TwitterOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/twitter/TwitterOAuth2Client.scala deleted file mode 100644 index fa575e8f6..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/twitter/TwitterOAuth2Client.scala +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.twitter - -import com.fasterxml.jackson.core.JsonProcessingException - -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.time.Instant -import java.util.Base64 -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import org.apache.hc.core5.net.URIBuilder -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.sources.bytestream.http.HttpClientException -import raw.utils.RawSettings - -import java.io.IOException - -object TwitterOAuth2Client { - - // Tokens requested using the PKCE flow can have the expiresIn, scope and refreshToken - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - case class TwitterAuth2TokenResponse( - accessToken: String, - tokenType: String, - expiresIn: Option[Long], - scope: Option[String], - refreshToken: Option[String] - ) - -} -class TwitterOAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - import TwitterOAuth2Client._ - - override def supportsRefreshToken: Boolean = false - - override def supportsClientCredentials: Boolean = true - - /** - * Executes the OAuth2 client credentials flow to obtain a new access token. Implementation is specific to the provider. - */ - override def newAccessTokenFromClientCredentials( - clientId: String, - clientSecret: String, - options: Map[String, String] - ): RenewedAccessToken = { - - try { - // https://developer.twitter.com/en/docs/authentication/api-reference/token - val uri = new URIBuilder("https://api.twitter.com/oauth2/token") - .addParameter("grant_type", "client_credentials") - .build() - - // twitter passes the client-id client-secret as basic auth - val strEncode = Base64.getEncoder.encodeToString(s"$clientId:$clientSecret".getBytes) - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .header("Authorization", s"Basic $strEncode") - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - logger.debug(s"Response status: ${response.statusCode()}") - - if (response.statusCode() == 200) { - val newToken = mapper.readValue[TwitterAuth2TokenResponse](response.body()) - // when passing the api-key/api-secret I always got the same token, which corresponds to the one given to the app - // which lasts forever, so if no expiration is defined then assume a long time (1 day) - val expiration: Instant = Instant.now().plusSeconds(newToken.expiresIn.getOrElse(24 * 3600)) - val scopes: Seq[String] = newToken.scope.map(_.split(" ")).getOrElse(Array.empty).toSeq - RenewedAccessToken(newToken.accessToken, expiration, scopes, newToken.refreshToken) - } else { - throw new CredentialsException(s"unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } catch { - case ex: IOException => throw new HttpClientException("HTTP error obtaining token from twitter", ex) - case ex: JsonProcessingException => throw new HttpClientException( - "error processing json response while getting token from twitter", - ex - ) - } - } -} diff --git a/sources/src/main/scala/raw/creds/oauth2/zoho/ZohoOAuth2Client.scala b/sources/src/main/scala/raw/creds/oauth2/zoho/ZohoOAuth2Client.scala deleted file mode 100644 index 31951a534..000000000 --- a/sources/src/main/scala/raw/creds/oauth2/zoho/ZohoOAuth2Client.scala +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2.zoho - -import java.io.IOException -import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpRequest, HttpResponse} -import java.nio.charset.StandardCharsets -import java.time.Instant -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.PropertyNamingStrategy -import com.fasterxml.jackson.databind.annotation.JsonNaming -import com.typesafe.scalalogging.StrictLogging -import org.apache.hc.core5.net.URIBuilder -import raw.creds.api.CredentialsException -import raw.creds.oauth2.api.{OAuth2Client, RenewedAccessToken} -import raw.utils.RawSettings - -object ZohoOAuth2Client { - - private val ACCOUNTS_URL = "accounts_url" - private val CLIENT_ID_KEY = "client_id" - private val CLIENT_SECRET_KEY = "client_secret" - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - private case class ZohoAuth2TokenResponse( - accessToken: String, - expiresIn: Int, - apiDomain: String, - tokenType: String, - refreshToken: Option[String], - error: Option[String] - ) - - @JsonNaming(classOf[PropertyNamingStrategy.SnakeCaseStrategy]) - case class ZohoDeviceCodeResponse( - userCode: String, - deviceCode: String, - verificationUriComplete: String, - verificationUrl: String, - interval: Int, - expiresIn: Int, - error: Option[String] - ) -} - -/** - * Zoho oauth client - * To use this class first create non-browser application in zoho api console. - * Then validated it with the client-protocol-flow as seen in zoho documentation: - * flow: https://www.zoho.com/accounts/protocol/oauth/devices/client-protocol-flow.html - * validation: https://www.zoho.com/accounts/protocol/oauth/devices/initiation-request.html - */ -class ZohoOAuth2Client(implicit settings: RawSettings) extends OAuth2Client with StrictLogging { - - import ZohoOAuth2Client._ - - override def supportsRefreshToken: Boolean = true - - override def supportsClientCredentials: Boolean = false - - override def newAccessTokenFromRefreshToken( - refreshToken: String, - options: Map[String, String] - ): RenewedAccessToken = { - // https://www.zoho.com/crm/developer/docs/api/v2/refresh.html - val accountsUrl = - options.getOrElse(ACCOUNTS_URL, throw new CredentialsException(s"""missing "$ACCOUNTS_URL" in options""")) - val clientId = - options.getOrElse(CLIENT_ID_KEY, throw new CredentialsException(s"""missing "$CLIENT_ID_KEY" in options""")) - val clientSecret = options.getOrElse( - CLIENT_SECRET_KEY, - throw new CredentialsException(s"""missing "$CLIENT_SECRET_KEY" in options""") - ) - - try { - val uri = new URIBuilder(accountsUrl) - .appendPath("/oauth/v2/token") - .addParameter("grant_type", "refresh_token") - .addParameter("refresh_token", refreshToken) - .addParameter("client_id", clientId) - .addParameter("client_secret", clientSecret) - .build() - - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - logger.debug(s"Sending request to ${request.uri()}") - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - logger.debug(s"Response status: ${response.statusCode()}") - - if (response.statusCode() == 200) { - val newToken = mapper.readValue[ZohoAuth2TokenResponse](response.body()) - - // Zoho will still return a 200 sometimes even though there was an error - // for example: '{"error": "invalid_client"}' - newToken.error.foreach(msg => - throw new CredentialsException( - s"error while trying to refresh token from ${accountsUrl + "/oauth/v2/token"}: $msg" - ) - ) - - RenewedAccessToken( - newToken.accessToken, - Instant.now().plusSeconds(newToken.expiresIn), - Seq.empty, - Some(refreshToken) - ) - } else { - throw new CredentialsException(s"unexpected response: ${response.statusCode()}. Error: ${response.body()}") - } - } catch { - case ex: JsonProcessingException => throw new CredentialsException( - s"error processing json response while refreshing zoho token from $accountsUrl", - ex - ) - case ex: IOException => throw new CredentialsException(s"error refreshing zoho token from $accountsUrl", ex) - } - } - - def getVerificationUrl( - clientId: String, - scope: String, - accountsUrl: String - ): ZohoDeviceCodeResponse = { - val uri = new URIBuilder(accountsUrl) - .appendPath("/oauth/v3/device/code") - .addParameter("client_id", clientId) - .addParameter("scope", scope) - .addParameter("grant_type", "device_request") - .addParameter("access_type", "offline") - .build() - - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - - if (response.statusCode() != 200) throw new CredentialsException( - s"unexpected response: ${response.statusCode()} while trying to get zoho authorization url" - ) - try { - val codeResponse = mapper.readValue[ZohoDeviceCodeResponse](response.body()) - codeResponse.error.foreach(msg => - throw new CredentialsException( - s"error getting verification url from ${accountsUrl + "/oauth/v3/device/code"}: $msg" - ) - ) - codeResponse - } catch { - case ex: JsonProcessingException => throw new CredentialsException( - s"error processing request from $accountsUrl: ${response.body()}", - ex - ) - } - } - - def exchangeCodeForRefreshToken( - clientId: String, - clientSecret: String, - accountsUrl: String, - code: String - ): RenewedAccessToken = { - val uri = new URIBuilder(accountsUrl) - .appendPath("/oauth/v3/device/token") - .addParameter("client_id", clientId) - .addParameter("client_secret", clientSecret) - .addParameter("code", code) - .addParameter("grant_type", "device_token") - .addParameter("access_type", "offline") - .build() - - val request = HttpRequest - .newBuilder() - .timeout(readTimeout) - .uri(uri) - .POST(HttpRequest.BodyPublishers.noBody()) - .build() - - val response: HttpResponse[String] = httpClient.send(request, BodyHandlers.ofString(StandardCharsets.UTF_8)) - - if (response.statusCode() != 200) throw new CredentialsException( - s"unexpected response: ${response.statusCode()} while trying to get zoho authorization url" - ) - try { - val token = mapper.readValue[ZohoAuth2TokenResponse](response.body()) - token.error.foreach(msg => - throw new CredentialsException( - s"error exchanging code for refresh token from ${accountsUrl + "/oauth/v3/device/token"}: $msg" - ) - ) - RenewedAccessToken( - token.accessToken, - Instant.now().plusSeconds(token.expiresIn), - Seq.empty, - token.refreshToken - ) - } catch { - case ex: JsonProcessingException => throw new CredentialsException( - s"error processing request from $accountsUrl: ${response.body()}", - ex - ) - } - } - -} diff --git a/sources/src/main/scala/raw/creds/protocol/ProtocolCredentials.scala b/sources/src/main/scala/raw/creds/protocol/ProtocolCredentials.scala deleted file mode 100644 index b8fcdb597..000000000 --- a/sources/src/main/scala/raw/creds/protocol/ProtocolCredentials.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.protocol - -import raw.creds.api._ -import raw.utils.AuthenticatedUser - -final case class RegisterNewHttpCredential(user: AuthenticatedUser, name: String, token: NewHttpCredential) -final case class GetNewHttpCredential(user: AuthenticatedUser, name: String) -final case class GetTokenNewHttpCredential(user: AuthenticatedUser, name: String) -final case class ListNewHttpCredentials(user: AuthenticatedUser) -final case class UnregisterNewHttpCredential(user: AuthenticatedUser, name: String) - -final case class RegisterDropboxCredential(user: AuthenticatedUser, token: DropboxToken) -final case class GetDropboxCredential(user: AuthenticatedUser) -final case class UnregisterDropboxCredential(user: AuthenticatedUser) - -final case class RegisterExternalConnectorCredential( - user: AuthenticatedUser, - name: String, - externalConnectorCredentialProtocol: ExternalConnectorCredential -) -final case class GetExternalConnectorCredential(user: AuthenticatedUser, name: String) -final case class ListExternalConnectorCredential(user: AuthenticatedUser) -final case class UnregisterExternalConnectorCredential(user: AuthenticatedUser, name: String) - -final case class RegisterHttpCredential(user: AuthenticatedUser, credential: HttpCredential) -final case class ListHttpCredentials(user: AuthenticatedUser) -final case class GetHttpCredential(user: AuthenticatedUser, name: String) -final case class UnregisterHttpCredential(user: AuthenticatedUser, name: String) - -final case class RegisterRelationalDatabaseCredential( - user: AuthenticatedUser, - name: String, - credential: RelationalDatabaseCredential -) -final case class GetRelationalDatabaseCredential(user: AuthenticatedUser, name: String) -final case class ListRelationalDatabaseCredentials(user: AuthenticatedUser) -final case class UnregisterRelationalDatabaseCredential(user: AuthenticatedUser, name: String) - -final case class RegisterS3BucketCredential(user: AuthenticatedUser, credential: S3Bucket) -final case class GetS3BucketCredential(user: AuthenticatedUser, name: String) -final case class ListS3BucketCredentials(user: AuthenticatedUser) -final case class UnregisterS3BucketCredential(user: AuthenticatedUser, name: String) - -final case class RegisterSecretCredential(user: AuthenticatedUser, credential: Secret) -final case class GetSecretCredential(user: AuthenticatedUser, name: String) -final case class ListSecretCredentials(user: AuthenticatedUser) -final case class UnregisterSecretCredential(user: AuthenticatedUser, name: String) - -final case class ProvisionFdwDbCredentials(user: AuthenticatedUser) diff --git a/sources/src/main/scala/raw/rest/client/ClientExceptions.scala b/sources/src/main/scala/raw/rest/client/ClientExceptions.scala deleted file mode 100644 index b871bb6d6..000000000 --- a/sources/src/main/scala/raw/rest/client/ClientExceptions.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.rest.client - -import raw.utils.RawException -import raw.rest.common.RestError - -/** - * Exceptions thrown by the REST Client. - */ -abstract class APIException(val message: String, cause: Throwable = null) extends RawException(message, cause) - -/** Exception thrown when the server cannot be reached. */ -final class ServerNotAvailableException(message: String, cause: Throwable = null) - extends APIException(s"server not available: $message", cause) - -/** Exception thrown when a bad or unexpected response is received. */ -final class BadResponseException(message: String, httpStatusCode: Int = -1) - extends APIException( - if (httpStatusCode > 0) s"bad response: $message (status code: $httpStatusCode)" - else s"bad response: $message" - ) - -/** Exception thrown when a request took too long to execute. */ -final class RequestTimeoutException extends APIException("request took too long to execute") - -/** - * Exception thrown for "logical" error condition. - * Logical error conditions are well-defined errors part of the protocol. - * The 'errorCode' is a well-defined string that can be "matched" to know the exact error condition. - * The 'message' contains a description of 'errorCode'. - * Corresponds to RestError in the REST protocol. - */ -class ClientAPIException(val errorCode: String, message: String) - extends APIException(s"API error: $message ($errorCode)") { - def this(restError: RestError) = this(restError.code, restError.message) -} diff --git a/sources/src/main/scala/raw/rest/client/RestClient.scala b/sources/src/main/scala/raw/rest/client/RestClient.scala deleted file mode 100644 index 516461df6..000000000 --- a/sources/src/main/scala/raw/rest/client/RestClient.scala +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.rest.client - -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.module.scala.JavaTypeable -import com.google.common.base.Stopwatch -import com.typesafe.scalalogging.StrictLogging -import org.apache.commons.io.IOUtils -import org.apache.hc.client5.http.classic.methods._ -import org.apache.hc.client5.http.config._ -import org.apache.hc.client5.http.impl.classic.{CloseableHttpResponse, HttpClientBuilder} -import org.apache.hc.client5.http.impl.io._ -import org.apache.hc.core5.http._ -import org.apache.hc.core5.http.io.entity.StringEntity -import org.apache.hc.core5.http.io.{HttpClientResponseHandler, SocketConfig} -import org.apache.hc.core5.net.URIBuilder -import org.apache.hc.core5.util.Timeout -import raw.auth.api.{ForbiddenException, GenericAuthException, TokenProvider, UnauthorizedException} -import raw.utils.RawSettings -import raw.rest.common._ - -import java.io.{IOException, InterruptedIOException} -import java.net.{URI, UnknownHostException} -import java.nio.charset.Charset -import java.time.Duration -import java.util.concurrent.TimeUnit -import scala.annotation.nowarn -import scala.concurrent.CancellationException - -object RestClient { - val X_RAW_CLIENT = "X-RAW-Client" - val X_RAW_CLIENT_VALUE = "Scala RAW REST Client/0.1" - - val mapper = new RestJsonMapper() - val restErrorReader = mapper.readerFor[GenericRestError] -} - -/** - * REST client used for RAW services. - * - * TODO (msb): This still needs some work in terms of exception handling, particularly stacktrace infos are very deep - * and not particularly useful. - */ -class RestClient( - serverHttpAddress: URI, - maybeTokenProvider: Option[TokenProvider], - name: String, - maybeImpersonateUser: Option[String] = None, - retryOnAccepted: Option[String] = - Some("/1/public/pending-request") // Legacy mode to automatically retry ACCEPTED responses in the following URL. -)(implicit settings: RawSettings) - extends StrictLogging { - - import RestClient._ - - // Remove any trailing /, as the URI class interprets a trailing slash as a segment of the path - private val cleanedServerHttpAddress = { - val uriStr = serverHttpAddress.toString - if (uriStr.endsWith("/")) { - new URI(uriStr.substring(0, uriStr.length - 1)) - } else { - serverHttpAddress - } - } - - logger.debug(s"[$name] Creating REST Client ($serverHttpAddress)") - private val asyncRequestRetries = settings.getInt("raw.rest.client.async-request-retries") - - private val serviceNotAvailableRetries = settings.getInt("raw.rest.client.service-not-available-retries") - private val serviceNotAvailableRetryIntervalMillis = - settings.getDuration("raw.rest.client.service-not-available-retry-interval", TimeUnit.MILLISECONDS) - - private val socketTimeout = settings.getDuration("raw.rest.client.socket-timeout", TimeUnit.MILLISECONDS).toInt - private val socketConfig = SocketConfig - .custom() - .setSoTimeout(Timeout.ofMilliseconds(socketTimeout)) - .build() - - private val connectTimeout = settings.getDuration("raw.rest.client.connect-timeout", TimeUnit.MILLISECONDS).toInt - - private val connectionConfig = ConnectionConfig - .custom() - .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout)) - .build() - - private val maxConnPerRoute = settings.getIntOpt("raw.rest.client.max-conn-per-route").getOrElse(20) - private val maxConnTotal = settings.getIntOpt("raw.rest.client.max-conn-total").getOrElse(100) - - private val connectionManager = PoolingHttpClientConnectionManagerBuilder - .create() - .setMaxConnPerRoute(maxConnPerRoute) - .setMaxConnTotal(maxConnTotal) - .setDefaultSocketConfig(socketConfig) - .setDefaultConnectionConfig(connectionConfig) - .build() - - private def checkAvailableConnections(): Unit = { - val stats = connectionManager.getTotalStats - if (stats.getPending > 0) { - logger.warn( - s"[$name] Some requests are blocked waiting for connections to become available: $stats" - ) - } - } - - private val requestConfig = RequestConfig - .custom() - .setResponseTimeout(Timeout.ofMilliseconds(socketTimeout)) - .build() - - private val httpClient = HttpClientBuilder - .create() - .setConnectionManager(connectionManager) - .setDefaultRequestConfig(requestConfig) - .build() - - def this(serverHttpAddress: URI, tokenProvider: TokenProvider, name: String)(implicit settings: RawSettings) = - this(serverHttpAddress, Some(tokenProvider), name) - - def this(serverHttpAddress: URI, name: String)(implicit settings: RawSettings) = this(serverHttpAddress, None, name) - - private def executeRequest(request: HttpUriRequest): CloseableHttpResponse = { - logger.trace(s"[$name] Sending request: ${request.getMethod} ${request.getUri}") - val start = Stopwatch.createStarted() - var response: CloseableHttpResponse = null - try { - checkAvailableConnections() - response = httpClient.execute(request) - } catch { - case ex @ (_: InterruptedIOException | _: CancellationException) => - logger.warn(s"[$name] Interrupted while waiting for response.", ex) - // If the I/O operation is interrupted in-flight because the thread doing it itself is interrupted, we get either - // InterruptedIOException or CancellationException (Apache HTTP Client). We convert it to InterruptedException so - // the rest of the system handles it as a normal interruption. - if (Thread.interrupted()) { - throw new InterruptedException() - } else { - throw new ServerNotAvailableException(s"error contacting $name", ex) - } - case ex: UnknownHostException => throw new ServerNotAvailableException(s"unknown host while contacting $name", ex) - case ex: IOException => throw new ServerNotAvailableException(s"error contacting $name", ex) - } finally { - logger.trace( - s"[$name] Request completed: ${request.getMethod} ${request.getUri}: " + - s"${if (response != null) response.getCode else "Failed"}. " + - s"Duration: ${start.elapsed(TimeUnit.MILLISECONDS)}ms" - ) - } - - throwExceptionIfErrorCondition(response) - } - - // TODO (ns) duplicate code from handleResponse - def handleUnexpected(response: ClassicHttpResponse): Exception = { - response.getCode match { - case statusCode @ HttpStatus.SC_BAD_REQUEST => - try { - readBody(response) match { - case Some(body) => - val ex = parseAPIException(statusCode, body) - throw ex - case None => new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case statusCode @ HttpStatus.SC_UNAUTHORIZED => - try { - readBody(response) match { - case Some(body) => new UnauthorizedException(body) - case None => new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case statusCode @ HttpStatus.SC_FORBIDDEN => - try { - readBody(response) match { - case Some(body) => new ForbiddenException(body) - case None => new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case statusCode if statusCode >= 500 => - try { - readBody(response) match { - case Some(body) => - new BadResponseException(s"${response.getCode} ${response.getReasonPhrase}\n" + body, statusCode) - case None => new BadResponseException(s"${response.getCode} ${response.getReasonPhrase}", statusCode) - } - } finally { - response.close() - } - case statusCode => - try { - readBody(response) match { - case Some(body) => new BadResponseException( - s"Unexpected response: ${response.getCode} ${response.getReasonPhrase}\n" + body.take(1024), - statusCode - ) - case None => new BadResponseException( - s"Unexpected response: ${response.getCode} ${response.getReasonPhrase}", - statusCode - ) - } - } finally { - response.close() - } - } - } - - private def throwExceptionIfErrorCondition[T <: ClassicHttpResponse](response: T): T = { - response.getCode match { - case statusCode @ HttpStatus.SC_BAD_REQUEST => - try { - readBody(response) match { - case Some(body) => - val ex = parseAPIException(statusCode, body) - throw ex - case None => throw new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case statusCode @ HttpStatus.SC_UNAUTHORIZED => - try { - readBody(response) match { - case Some(body) => throw new UnauthorizedException(body) - case None => throw new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case statusCode @ HttpStatus.SC_FORBIDDEN => - try { - readBody(response) match { - case Some(body) => throw new ForbiddenException(body) - case None => throw new BadResponseException("invalid body", statusCode) - } - } finally { - response.close() - } - case HttpStatus.SC_SERVICE_UNAVAILABLE => response - case statusCode if statusCode >= 500 => - try { - readBody(response) match { - case Some(body) => - throw new BadResponseException(s"${response.getCode} ${response.getReasonPhrase}\n" + body, statusCode) - case None => throw new BadResponseException(s"${response.getCode} ${response.getReasonPhrase}", statusCode) - } - } finally { - response.close() - } - case _ => response - } - } - - /** - * Method for clients to override and parse the body and status code in different manners. - * Must throw a (type of) ClientAPIException. - */ - protected def parseAPIException(statusCode: Int, body: String): Exception = { - try { - val restError = restErrorReader.readValue[GenericRestError](body) - new ClientAPIException(restError) - } catch { - case ex: JsonProcessingException => - logger.debug("Client received bad response.", ex) - new BadResponseException(body, statusCode) - } - } - - def readBody(response: ClassicHttpResponse): Option[String] = { - if (response.getEntity == null) { - None - } else { - val resp = - try { - IOUtils.toString(response.getEntity.getContent, Charset.forName("UTF-8")).trim - } catch { - case ex: IOException => - // TODO (msb): Questionable choice of exception. Use BadResponseException instead? - throw new ServerNotAvailableException(s"error reading response body from $name", ex) - } - if (resp.isEmpty) { - None - } else { - Some(resp) - } - } - } - - private def setJsonPayload(request: HttpUriRequestBase, payload: Any): Unit = { - request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType) - val reqBody = mapper.writeValueAsString(payload) - request.setEntity(new StringEntity(reqBody, ContentType.APPLICATION_JSON)) - } - - private def setPlainTextPayload(request: HttpUriRequestBase, payload: String): Unit = { - request.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.TEXT_PLAIN.getMimeType) - request.setEntity(new StringEntity(payload)) - } - - private def doJsonRequestWithEmptyResponse( - request: HttpUriRequestBase, - payload: Any, - expectedStatus: Int, - queryHeaders: Seq[(String, String)] - ): Unit = { - setJsonPayload(request, payload) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - val maybeBody = readBody(response) - if (maybeBody.isDefined) { - throw new BadResponseException("expected empty body on response", statusCode) - } - } finally { - response.close() - } - } - - private def doJsonRequestWithResponse[T]( - request: HttpUriRequestBase, - payload: Any, - expectedStatus: Int, - queryHeaders: Seq[(String, String)] - )( - implicit classTag: JavaTypeable[T] - ): T = { - setJsonPayload(request, payload) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - val body = - readBody(response).getOrElse(throw new BadResponseException("expected non-empty body on response", statusCode)) - mapper.readValue[T](body) - } finally { - response.close() - } - } - - private def executeInternal[T]( - request: ClassicHttpRequest, - responseHandler: HttpClientResponseHandler[T], - @nowarn retrialsLeft: Int = serviceNotAvailableRetries - )( - implicit classTag: JavaTypeable[T] - ): T = { - logger.trace(s"[$name] Sending request: ${request.getMethod} ${request.getUri}}") - val start = Stopwatch.createStarted() - val response = - try { - checkAvailableConnections() - httpClient.execute(request) - } catch { - case ex @ (_: InterruptedIOException | _: CancellationException) => - logger.warn(s"[$name] Interrupted while waiting for response.", ex) - // If the I/O operation is interrupted in-flight because the thread doing it itself is interrupted, we get either - // InterruptedIOException or CancellationException (Apache HTTP Client). We convert it to InterruptedException so - // the rest of the system handles it as a normal interruption. - if (Thread.interrupted()) { - throw new InterruptedException() - } else { - throw new ServerNotAvailableException(s"error contacting $name", ex) - } - case ex: UnknownHostException => - throw new ServerNotAvailableException(s"unknown host while contacting $name", ex) - case ex: IOException => throw new ServerNotAvailableException(s"error contacting $name", ex) - } finally { - logger.trace( - s"[$name] Request completed: ${request.getMethod} ${request.getUri}" + - s"Duration: ${start.elapsed(TimeUnit.MILLISECONDS)}ms" - ) - } - if (retrialsLeft > 0 && response.getCode == HttpStatus.SC_SERVICE_UNAVAILABLE) { - response.close() - logger.debug( - s"[$name] Service temporarily unavailable. Retrying in $serviceNotAvailableRetryIntervalMillis milliseconds." - ) - Thread.sleep(serviceNotAvailableRetryIntervalMillis) - executeInternal(request, responseHandler, retrialsLeft - 1) - } else { - try { - throwExceptionIfErrorCondition(response) - responseHandler.handleResponse(response) - } finally { - response.close() - } - } - } - - def newGet(path: String, queryParams: Map[String, Any] = Map.empty, withAuth: Boolean = true): HttpGet = { - val uri = buildUri(path, queryParams) - val httpGet = new HttpGet(uri) - configureRequest(httpGet, withAuth) - httpGet - } - - def newPost(path: String, queryParams: Map[String, Any] = Map.empty, withAuth: Boolean = true): HttpPost = { - val uri = buildUri(path, queryParams) - val httpPost = new HttpPost(uri) - configureRequest(httpPost, withAuth) - httpPost - } - - def newPut(path: String, queryParams: Map[String, Any] = Map.empty, withAuth: Boolean = true): HttpPut = { - val uri = buildUri(path, queryParams) - val httpPut = new HttpPut(uri) - configureRequest(httpPut, withAuth) - httpPut - } - - def newDelete(path: String, queryParams: Map[String, Any] = Map.empty, withAuth: Boolean = true): HttpDelete = { - val uri = buildUri(path, queryParams) - val httpDelete = new HttpDelete(uri) - configureRequest(httpDelete, withAuth) - httpDelete - } - - private def buildUri(path: String, queryParams: Map[String, Any]): URI = { - val uriBuilder = new URIBuilder(cleanedServerHttpAddress) - .appendPath(path.replaceAllLiterally("|", "%7C")) - queryParams.foreach { - case (k, v) => v match { - case s: String => uriBuilder.addParameter(k, s) - case i: Int => uriBuilder.addParameter(k, i.toString) - case l: Long => uriBuilder.addParameter(k, l.toString) - case d: Duration => uriBuilder.addParameter(k, d.toString) // java.time.Duration converts to ISO-8601 Duration - case b: Boolean => uriBuilder.addParameter(k, b.toString) - case Some(s: String) => uriBuilder.addParameter(k, s) - // CTM: another option is to not add the parameter - case None => uriBuilder.addParameter(k, null) - case l: List[_] => - // TODO (msb): This only works for List[String]. BTW, erasure absolutely sucks... - uriBuilder.addParameter(k, l.map(_.toString).mkString(",")) - } - } - uriBuilder.build - } - - private def configureRequest(req: HttpUriRequestBase, withAuth: Boolean): Unit = { - req.setHeader(RestClient.X_RAW_CLIENT, RestClient.X_RAW_CLIENT_VALUE) - // FIXME: Move this to be a developer mode settings - // req.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity") // No gzip - if (withAuth) { - maybeTokenProvider - .map { tokenProvider => - val accessToken = - try { - tokenProvider.token.accessToken - } catch { - case ex: GenericAuthException => - throw new ServerNotAvailableException("cannot retrieve access token from token provider", ex) - } - req.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken) - } - .getOrElse { - throw new ServerNotAvailableException("no token provider specified but authentication required") - } - maybeImpersonateUser.foreach(uid => req.setHeader(RawHttpHeaders.X_IMPERSONATE_UID, uid)) - } - } - - /** General purpose */ - - def doRequest(req: HttpUriRequestBase, queryHeaders: Seq[(String, String)] = Seq.empty): CloseableHttpResponse = { - queryHeaders.foreach { case (k, v) => req.setHeader(k, v) } - var response = executeRequest(req) - var retriesLeft = serviceNotAvailableRetries - while (retriesLeft > 0 && response.getCode == HttpStatus.SC_SERVICE_UNAVAILABLE) { - response.close() - logger.debug( - s"[$name] Service temporarily unavailable. Retrying in $serviceNotAvailableRetryIntervalMillis milliseconds. Retries left: $retriesLeft." - ) - Thread.sleep(serviceNotAvailableRetryIntervalMillis) - retriesLeft -= 1 - response = executeRequest(req) - } - if (response.getCode == HttpStatus.SC_SERVICE_UNAVAILABLE) { - throw new RequestTimeoutException() - } - - retryOnAccepted match { - case Some(urlToRetry) => - // Legacy mode where requests were accepted and this REST Client would automatically retry them transparently. - // Kept for backwards compatibility but not necessary anymore. - retriesLeft = asyncRequestRetries - while (retriesLeft > 0 && response.getCode == HttpStatus.SC_ACCEPTED) { - val requestUuid = readBody(response) match { - case Some(str) => str - case None => throw new BadResponseException("Received a 202 response without a request ID") - } - response.close() - logger.debug( - s"[$name] Request did not complete before timeout. Retrying with request id: $requestUuid. Retries left: $retriesLeft." - ) - val retryRequest = newPost(urlToRetry) - setPlainTextPayload(retryRequest, requestUuid) - retriesLeft -= 1 - response = executeRequest(retryRequest) - } - case None => - - } - if (response.getCode == HttpStatus.SC_ACCEPTED) { - throw new RequestTimeoutException() - } - response - } - - /** GET */ - - def doGet[T]( - path: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - )( - implicit classTag: JavaTypeable[T] - ): T = { - val request = newGet(path, queryParams, withAuth) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus", statusCode) - } - val body = - readBody(response).getOrElse(throw new BadResponseException("expected non-empty body on response", statusCode)) - mapper.readValue[T](body) - } finally { - response.close() - } - } - - def doGetWithPlainTextResponse( - path: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): String = { - val request = newGet(path, queryParams, withAuth) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus", statusCode) - } - readBody(response).getOrElse(throw new BadResponseException("expected non-empty body on response", statusCode)) - } finally { - response.close() - } - } - - def doGetWithEmptyResponse( - path: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): Unit = { - val request = newGet(path, queryParams, withAuth) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus", statusCode) - } - val maybeBody = readBody(response) - if (maybeBody.isDefined) { - throw new BadResponseException("expected empty body on response", statusCode) - } - } finally { - response.close() - } - } - - /** POST */ - - def doPlainTextPost[T]( - path: String, - payload: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - )( - implicit classTag: JavaTypeable[T] - ): T = { - val request = newPost(path, queryParams, withAuth) - setPlainTextPayload(request, payload) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - val body = - readBody(response).getOrElse(throw new BadResponseException("expected non-empty body on response", statusCode)) - mapper.readValue[T](body) - } finally { - response.close() - } - } - - // Used to pass response back to consumer, who may close it later. - def doPlainTextPostWithOpenResponse( - path: String, - maybePayload: Option[String] = None, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): CloseableHttpResponse = { - val request = newPost(path, queryParams, withAuth) - maybePayload.foreach(payload => setPlainTextPayload(request, payload)) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - if (statusCode != expectedStatus) { - response.close() - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - response - } - - def doJsonPostWithEmptyResponse( - path: String, - payload: Any, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): Unit = { - val post = newPost(path, queryParams, withAuth) - doJsonRequestWithEmptyResponse(post, payload, expectedStatus, queryHeaders) - } - - def doJsonPost[T]( - path: String, - payload: Any, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - )( - implicit classTag: JavaTypeable[T] - ): T = { - val post = newPost(path, queryParams, withAuth) - doJsonRequestWithResponse(post, payload, expectedStatus, queryHeaders) - } - - /** PUT */ - - def doJsonPut[T]( - path: String, - payload: Any, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - )( - implicit classTag: JavaTypeable[T] - ): T = { - val req = newPut(path, queryParams, withAuth) - doJsonRequestWithResponse(req, payload, expectedStatus, queryHeaders) - } - - def doPlainTextPut[T]( - path: String, - payload: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - )( - implicit classTag: JavaTypeable[T] - ): T = { - val request = newPut(path, queryParams, withAuth) - setPlainTextPayload(request, payload) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - val body = - readBody(response).getOrElse(throw new BadResponseException("expected non-empty body on response", statusCode)) - mapper.readValue[T](body) - } finally { - response.close() - } - } - - // TODO (msb): expectedStatus should be SC_NO_CONTENT for empty responses? - def doPlainTextPutWithEmptyResponse( - path: String, - payload: String, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): Unit = { - val request = newPut(path, queryParams, withAuth) - setPlainTextPayload(request, payload) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus but got $statusCode", statusCode) - } - val maybeBody = readBody(response) - if (maybeBody.isDefined) { - throw new BadResponseException("expected empty body on response", statusCode) - } - } finally { - response.close() - } - } - - // TODO (msb): expectedStatus sohuld be SC_NO_CONTENT for empty responses? - def doJsonPutWithEmptyResponse( - path: String, - payload: Any, - expectedStatus: Int = HttpStatus.SC_OK, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): Unit = { - val request = newPut(path, queryParams, withAuth) - doJsonRequestWithEmptyResponse(request, payload, expectedStatus, queryHeaders) - } - - /** DELETE */ - - def doDeleteWithEmptyResponse( - path: String, - expectedStatus: Int = HttpStatus.SC_NO_CONTENT, - queryHeaders: Seq[(String, String)] = Seq.empty, - queryParams: Map[String, Any] = Map.empty, - withAuth: Boolean = true - ): Unit = { - val request = newDelete(path, queryParams, withAuth) - val response = doRequest(request, queryHeaders) - val statusCode = response.getCode - try { - if (statusCode != expectedStatus) { - throw new BadResponseException(s"expected response status code $expectedStatus", statusCode) - } - val maybeBody = readBody(response) - if (maybeBody.isDefined) { - throw new BadResponseException("expected empty body on response", statusCode) - } - } finally { - response.close() - } - } - - /** Service Calls */ - - def version(): String = { - doGetWithPlainTextResponse("version", expectedStatus = HttpStatus.SC_OK, withAuth = false) - } - - def health(): Unit = { - doGetWithEmptyResponse("health", expectedStatus = HttpStatus.SC_NO_CONTENT, withAuth = false) - } - - def close(): Unit = { - logger.debug(s"[$name] Closing REST Client ($serverHttpAddress)") - httpClient.close() - connectionManager.close() - } - -} diff --git a/sources/src/main/scala/raw/rest/common/RawHttpHeaders.scala b/sources/src/main/scala/raw/rest/common/RawHttpHeaders.scala deleted file mode 100644 index bb51c6366..000000000 --- a/sources/src/main/scala/raw/rest/common/RawHttpHeaders.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.rest.common - -object RawHttpHeaders { - val X_IMPERSONATE_UID = "X-Impersonate-Uid" - val X_IMPERSONATE_USERNAME = "X-Impersonate-Username" - val X_ID_TOKEN = "X-Id-Token" -} diff --git a/sources/src/main/scala/raw/rest/common/RestError.scala b/sources/src/main/scala/raw/rest/common/RestError.scala deleted file mode 100644 index f7eead69e..000000000 --- a/sources/src/main/scala/raw/rest/common/RestError.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.rest.common - -// Base interface for REST errors. -// Every REST error has a code as a string (e.g. "thisHappened") and a user-visible message. -trait RestError { - def code: String - - def message: String -} - -final case class GenericRestError(code: String, message: String) extends RestError diff --git a/sources/src/main/scala/raw/rest/common/RestJsonMapper.scala b/sources/src/main/scala/raw/rest/common/RestJsonMapper.scala deleted file mode 100644 index 69ce250c6..000000000 --- a/sources/src/main/scala/raw/rest/common/RestJsonMapper.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.rest.common - -import com.fasterxml.jackson.annotation.JsonInclude.Include -import com.fasterxml.jackson.databind._ -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} - -// failOnUnknownProperties set to false by default to ease forward-compatibility. -class RestJsonMapper(failOnUnknownProperties: Boolean = false) extends ObjectMapper with ClassTagExtensions { - registerModule(new JavaTimeModule()) - registerModule(new Jdk8Module()) - registerModule(DefaultScalaModule) - configure(SerializationFeature.INDENT_OUTPUT, false) - configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, failOnUnknownProperties) - // The default mapper will not include properties with null or None value. - // We include these properties to simplify the flow of the clients. - setSerializationInclusion(Include.ALWAYS) - configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false) -} diff --git a/sources/src/main/scala/raw/sources/api/Location.scala b/sources/src/main/scala/raw/sources/api/Location.scala index 3a7976742..70e1fa1eb 100644 --- a/sources/src/main/scala/raw/sources/api/Location.scala +++ b/sources/src/main/scala/raw/sources/api/Location.scala @@ -12,14 +12,8 @@ package raw.sources.api -import com.typesafe.scalalogging.StrictLogging - -trait Location extends StrictLogging { - - def rawUri: String +trait Location { def testAccess(): Unit - override def toString: String = rawUri - } diff --git a/sources/src/main/scala/raw/sources/api/LocationBuilder.scala b/sources/src/main/scala/raw/sources/api/LocationBuilder.scala deleted file mode 100644 index 95475024f..000000000 --- a/sources/src/main/scala/raw/sources/api/LocationBuilder.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.api - -import raw.client.api.LocationDescription - -trait LocationBuilder { - - def schemes: Seq[String] - - def build(location: LocationDescription)(implicit sourceContext: SourceContext): Location - -} diff --git a/sources/src/main/scala/raw/sources/api/LocationProvider.scala b/sources/src/main/scala/raw/sources/api/LocationProvider.scala deleted file mode 100644 index 0c8704161..000000000 --- a/sources/src/main/scala/raw/sources/api/LocationProvider.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.api - -import raw.client.api.LocationDescription - -trait LocationProvider { - - def build(location: LocationDescription)(implicit sourceContext: SourceContext): Location - - protected def getScheme(url: String): Option[String] = { - val i = url.indexOf(':') - if (i == -1) None - else Some(url.take(i)) - } - -} diff --git a/sources/src/main/scala/raw/sources/api/SourceContext.scala b/sources/src/main/scala/raw/sources/api/SourceContext.scala deleted file mode 100644 index 8c4175ae0..000000000 --- a/sources/src/main/scala/raw/sources/api/SourceContext.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.api - -import raw.utils.{AuthenticatedUser, RawSettings} -import raw.creds.api.CredentialsService -import raw.sources.bytestream.api.ByteStreamLocationBuilder -import raw.sources.filesystem.api.FileSystemLocationBuilder -import raw.sources.jdbc.api.{JdbcLocationBuilder, JdbcSchemaLocationBuilder, JdbcTableLocationBuilder} - -class SourceContext( - val user: AuthenticatedUser, - val credentialsService: CredentialsService, - val settings: RawSettings -) { - - val byteStreamLocationBuilderServices: Array[ByteStreamLocationBuilder] = Array( - new raw.sources.filesystem.local.LocalFileSystemLocationBuilder, - new raw.sources.filesystem.dropbox.DropboxFileSystemLocationBuilder, - new raw.sources.bytestream.github.GithubByteStreamLocationBuilder, - new raw.sources.bytestream.http.HttpByteStreamLocationBuilder, - new raw.sources.bytestream.in_memory.InMemoryByteStreamLocationBuilder, - new raw.sources.filesystem.mock.MockFileSystemLocationBuilder, - new raw.sources.filesystem.s3.S3FileSystemLocationBuilder - ) - - val fileSystemLocationBuilderServices: Array[FileSystemLocationBuilder] = Array( - new raw.sources.filesystem.local.LocalFileSystemLocationBuilder, - new raw.sources.filesystem.dropbox.DropboxFileSystemLocationBuilder, - new raw.sources.filesystem.mock.MockFileSystemLocationBuilder, - new raw.sources.filesystem.s3.S3FileSystemLocationBuilder - ) - - val jdbcLocationBuilderServices: Array[JdbcLocationBuilder] = Array( - new raw.sources.jdbc.sqlite.SqliteLocationBuilder, - new raw.sources.jdbc.snowflake.SnowflakeLocationBuilder, - new raw.sources.jdbc.pgsql.PostgresqlLocationBuilder, - new raw.sources.jdbc.mysql.MySqlLocationBuilder, - new raw.sources.jdbc.sqlserver.SqlServerLocationBuilder, - new raw.sources.jdbc.oracle.OracleLocationBuilder, - new raw.sources.jdbc.teradata.TeradataLocationBuilder - ) - - val jdbcSchemaLocationBuilderServices: Array[JdbcSchemaLocationBuilder] = Array( - new raw.sources.jdbc.sqlite.SqliteSchemaLocationBuilder, - new raw.sources.jdbc.snowflake.SnowflakeSchemaLocationBuilder, - new raw.sources.jdbc.pgsql.PostgresqlSchemaLocationBuilder, - new raw.sources.jdbc.mysql.MySqlSchemaLocationBuilder, - new raw.sources.jdbc.sqlserver.SqlServerSchemaLocationBuilder, - new raw.sources.jdbc.oracle.OracleSchemaLocationBuilder, - new raw.sources.jdbc.teradata.TeradataSchemaLocationBuilder - ) - - val jdbcTableLocationBuilderServices: Array[JdbcTableLocationBuilder] = Array( - new raw.sources.jdbc.sqlite.SqliteTableLocationBuilder, - new raw.sources.jdbc.snowflake.SnowflakeTableLocationBuilder, - new raw.sources.jdbc.pgsql.PostgresqlTableLocationBuilder, - new raw.sources.jdbc.mysql.MySqlTableLocationBuilder, - new raw.sources.jdbc.sqlserver.SqlServerTableLocationBuilder, - new raw.sources.jdbc.oracle.OracleTableLocationBuilder, - new raw.sources.jdbc.teradata.TeradataTableLocationBuilder - ) - -} diff --git a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocation.scala b/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocation.scala index 38d8d6572..0f23fe246 100644 --- a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocation.scala +++ b/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocation.scala @@ -17,7 +17,6 @@ import java.nio.charset.{Charset, StandardCharsets} import java.nio.file.Path import org.apache.commons.io.ByteOrderMark import org.apache.commons.io.input.BOMInputStream -import raw.utils.RawException import raw.sources.api._ import scala.util.control.NonFatal @@ -25,7 +24,6 @@ import scala.util.control.NonFatal trait ByteStreamLocation extends Location { // This call uses the retry mechanism. - @throws[RawException] final def getInputStream: InputStream = { doGetInputStream() } @@ -39,7 +37,6 @@ trait ByteStreamLocation extends Location { protected def doGetSeekableInputStream(): SeekableInputStream - @throws[RawException] final def getReader(encoding: Encoding): Reader = { val charset = encoding.charset val is = getInputStream @@ -76,7 +73,11 @@ trait ByteStreamLocation extends Location { } } - // TODO (msb): This belongs to the file system location API (or shouldn't exist at all?) + /** + * Caches the content of the location in a local file and returns the path to it. + * + * @return the path to the local file + */ def getLocalPath(): Path } diff --git a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationBuilder.scala b/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationBuilder.scala deleted file mode 100644 index 5c14645c3..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.api - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationBuilder, SourceContext} - -trait ByteStreamLocationBuilder extends LocationBuilder { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): ByteStreamLocation - -} diff --git a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationProvider.scala b/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationProvider.scala deleted file mode 100644 index e49944fdb..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/api/ByteStreamLocationProvider.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.api - -import raw.client.api.LocationDescription -import raw.utils.RawException -import raw.sources.api._ - -object ByteStreamLocationProvider extends LocationProvider { - - def isSupported(location: LocationDescription)(implicit sourceContext: SourceContext): Boolean = { - isSupported(location.url) - } - - def isSupported(url: String)(implicit sourceContext: SourceContext): Boolean = { - getScheme(url) match { - case Some(scheme) => sourceContext.byteStreamLocationBuilderServices.exists(_.schemes.contains(scheme)) - case None => false - } - } - - @throws[RawException] - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): ByteStreamLocation = { - getScheme(location.url) match { - case Some(scheme) => - val impls = sourceContext.byteStreamLocationBuilderServices.filter(_.schemes.contains(scheme)) - if (impls.isEmpty) throw new ByteStreamException(s"no byte stream location implementation found for $scheme") - else if (impls.length > 1) - throw new ByteStreamException(s"more than one byte stream location implementation found for $scheme") - else impls.head.build(location) - case None => throw new ByteStreamException(s"invalid url: '${location.url}'") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/bytestream/github/GitHubLocation.scala b/sources/src/main/scala/raw/sources/bytestream/github/GitHubLocation.scala new file mode 100644 index 000000000..525ffc90d --- /dev/null +++ b/sources/src/main/scala/raw/sources/bytestream/github/GitHubLocation.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.bytestream.github + +import java.io.InputStream +import java.nio.file.Path +import com.typesafe.scalalogging.StrictLogging +import raw.sources.bytestream.http.{HttpByteStreamException, HttpByteStreamLocation} +import raw.sources.bytestream.api.{ByteStreamException, ByteStreamLocation, SeekableInputStream} +import raw.utils.RawSettings + +// Supports only public repositories. +class GitHubLocation(val username: String, val repo: String, val file: String, val maybeBranch: Option[String])( + implicit settings: RawSettings +) extends ByteStreamLocation + with StrictLogging { + + // If branch is not defined, try to find the default one. + // Tried listing branches like this: + // curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/torcato/test-repo/branches + // But got the following after some tests: + // {"message":"API rate limit exceeded for 84.226.22.197. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"} + private val branch = { + maybeBranch.getOrElse { + + def testBranch(branch: String) = { + val githubUrl = s"https://github.com/$username/$repo/tree/$branch" + try { + val httpLocation = new HttpByteStreamLocation(githubUrl) + httpLocation.testAccess() + true + } catch { + case _: HttpByteStreamException => false + } + } + + Seq("main", "master") // Default branch names. + .find(testBranch) + .getOrElse( + throw new ByteStreamException( + s"could not find default branch after trying 'main' and 'master'; is the GitHub repository public?" + ) + ) + } + } + + private val githubUrl = s"https://github.com/$username/$repo/tree/$branch/$file" + + private val httpClient = new HttpByteStreamLocation(githubUrl) + + override protected def doGetInputStream(): InputStream = { + httpClient.getInputStream + } + + override protected def doGetSeekableInputStream(): SeekableInputStream = { + httpClient.getSeekableInputStream + } + + override def getLocalPath(): Path = { + httpClient.getLocalPath() + } + + override def testAccess(): Unit = { + httpClient.testAccess() + } +} diff --git a/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocation.scala b/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocation.scala deleted file mode 100644 index 1e47584a4..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocation.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.github - -import java.io.InputStream -import java.nio.file.Path -import com.typesafe.scalalogging.StrictLogging -import raw.sources.bytestream.http.HttpByteStreamLocation -import raw.sources.bytestream.api.{ByteStreamLocation, SeekableInputStream} - -class GithubByteStreamLocation( - http: HttpByteStreamLocation, - url: String -) extends ByteStreamLocation - with StrictLogging { - - override protected def doGetInputStream(): InputStream = http.getInputStream - - override protected def doGetSeekableInputStream(): SeekableInputStream = http.getSeekableInputStream - - override def getLocalPath(): Path = http.getLocalPath() - - override def rawUri: String = url - - override def testAccess(): Unit = { - http.testAccess() - } -} diff --git a/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocationBuilder.scala b/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocationBuilder.scala deleted file mode 100644 index 308d79c6e..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/github/GithubByteStreamLocationBuilder.scala +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.github - -import com.typesafe.scalalogging.StrictLogging -import raw.sources.bytestream.http.{HttpByteStreamLocationBuilder, HttpClientException} -import raw.sources.bytestream.api.{ByteStreamLocation, ByteStreamLocationBuilder} -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -class GithubByteStreamLocationBuilder extends ByteStreamLocationBuilder with StrictLogging { - - // Github regex github:////[\[\]] - // branch is optional for example: - // github://torcato/test-repo/1/public/code.rql the same as github://torcato/test-repo/1/public/code.rql[main] - private val githubRegex = """github://([^/]+)/([^/]+)/(?:([^\[]+)\[(.*)\]|(.*))""".r - val httpBuilder = new HttpByteStreamLocationBuilder() - - override def schemes: Seq[String] = Seq("github") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): ByteStreamLocation = { - location.url match { - case githubRegex(username, repo, maybeNullFile, maybeNullBranch, maybeNullFileWithoutBranch) => - val (file: String, branch: String) = - if (maybeNullBranch != null) { - (maybeNullFile, maybeNullBranch) - } else { - def testBranch(branch: String) = { - val url = s"https://github.com/$username/$repo/tree/$branch" - try { - val httpLocation = httpBuilder.build(LocationDescription(url, location.settings)) - httpLocation.testAccess() - true - } catch { - case _: HttpClientException => false - } - } - // tried listing branches like this - // curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/torcato/test-repo/branches - // but got the following after some tests - // {"message":"API rate limit exceeded for 84.226.22.197. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"} - val branch = Seq("main", "master") - .find(testBranch) - .getOrElse( - throw new LocationException( - s"could not find default branch for ${location.url}, tried (main, master), is the github repository public?" - ) - ) - - (maybeNullFileWithoutBranch, branch) - } - val url = s"https://raw.githubusercontent.com/$username/$repo/$branch/$file" - val httpLocation = httpBuilder.build(LocationDescription(url, location.settings)) - new GithubByteStreamLocation(httpLocation, location.url) - case _ => throw new LocationException("not an Github location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/bytestream/http/ApacheRuntimeHttpClient.scala b/sources/src/main/scala/raw/sources/bytestream/http/ApacheRuntimeHttpClient.scala deleted file mode 100644 index f5b97cbcb..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/http/ApacheRuntimeHttpClient.scala +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.http - -import com.typesafe.scalalogging.StrictLogging -import org.apache.hc.client5.http.classic.methods._ -import org.apache.hc.client5.http.config.RequestConfig -import org.apache.hc.client5.http.impl.classic.{CloseableHttpClient, CloseableHttpResponse, HttpClients} -import org.apache.hc.client5.http.impl.io.{ - PoolingHttpClientConnectionManager, - PoolingHttpClientConnectionManagerBuilder -} -import org.apache.hc.core5.http.ContentType -import org.apache.hc.core5.http.io.SocketConfig -import org.apache.hc.core5.http.io.entity.ByteArrayEntity -import org.apache.hc.core5.net.URIBuilder -import org.apache.hc.core5.util.{TimeValue, Timeout} -import raw.sources.api.LocationException -import raw.sources.bytestream.api._ -import raw.utils._ - -import java.io.{ByteArrayInputStream, IOException, InputStream} -import java.net._ -import java.util.concurrent.{Executors, TimeUnit} - -final class HttpClientException(message: String, cause: Throwable = null) - extends ByteStreamException(s"http error: $message", cause) - -object HttpClientSettings { - val HTTP_READ_TIMEOUT = "raw.sources.bytestream.http.read-timeout" - val HTTP_CONNECT_TIMEOUT = "raw.sources.bytestream.http.connect-timeout" -} - -object ApacheRuntimeHttpClient { - - import HttpClientSettings._ - - private val HTTP_CONN_POOL_MAX_TOTAL = "raw.sources.bytestream.http.conn-pool-max-total" - private val HTTP_CONN_POOL_MAX_PER_ROUTE = "raw.sources.bytestream.http.conn-pool-max-per-route" - - private var apacheHttpClient: ApacheHttpClientHolder = _ - private val apacheHttpClientInitLock = new Object - - protected[http] def buildApacheHttpClient(settings: RawSettings): ApacheHttpClientHolder = synchronized { - apacheHttpClientInitLock.synchronized { - // Global Apache Http Client for all instances of HttpClient - if (apacheHttpClient == null) { - val httpReadTimeout = settings.getDuration(HTTP_READ_TIMEOUT, TimeUnit.MILLISECONDS) - val socketConfig = SocketConfig - .custom() - .setSoTimeout(Timeout.ofMilliseconds(httpReadTimeout)) - .build() - - val pool = PoolingHttpClientConnectionManagerBuilder - .create() - .setDefaultSocketConfig(socketConfig) - .setMaxConnTotal(settings.getInt(HTTP_CONN_POOL_MAX_TOTAL)) - .setMaxConnPerRoute(settings.getInt(HTTP_CONN_POOL_MAX_PER_ROUTE)) - .build() - - val client = HttpClients - .custom() - .setConnectionManager(pool) - // setConnectionManagerShared(true) is needed to avoid a bug where the connection manager was closed when an - // unexpected exception occurred. - .setConnectionManagerShared(true) - .build() - apacheHttpClient = new ApacheHttpClientHolder(client, pool) - - Executors - .newSingleThreadScheduledExecutor(RawUtils.newThreadFactory("http-client-idle-connections-cleanup")) - .scheduleAtFixedRate(() => { apacheHttpClient.closeIdleConnections() }, 1, 1, TimeUnit.MINUTES) - } - } - apacheHttpClient - } -} - -private[http] class ApacheHttpClientHolder( - val httpClient: CloseableHttpClient, - connPool: PoolingHttpClientConnectionManager -) { - - def closeIdleConnections(): Unit = { - connPool.closeIdle(TimeValue.ofMinutes(30)) - connPool.closeExpired() - } -} - -class ApacheRuntimeHttpClient( - method: String, - args: Array[(String, String)], - headers: Array[(String, String)], - body: Option[Array[Byte]] -)(implicit settings: RawSettings) - extends InputStreamClient - with StrictLogging - with RuntimeHttpClient { - - import HttpClientSettings._ - - private val apacheHttpClient = ApacheRuntimeHttpClient.buildApacheHttpClient(settings) - - private val requestConfig = { - val httpConnectTimeout = settings.getDuration(HTTP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) - RequestConfig - .custom() - .setConnectTimeout(Timeout.ofMilliseconds(httpConnectTimeout.toInt)) - .build() - } - - // This method is called by our runtime in a read so expects the status code to be 200 - override def getInputStream(url: String): InputStream = { - val (response, request) = openHTTPConnection(url) - try { - val responseCode = response.getCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val is = response.getEntity.getContent - val body = - try { - readOutputBounded(response.getEntity.getContent) - } finally { - is.close() - } - if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw new HttpClientException(s"authorization error accessing $url: ($responseCode)\n$body") - } else { - throw new HttpClientException( - s"could not read (HTTP ${method.toUpperCase}) from $url: ($responseCode)\n$body" - ) - } - } - // The response is wrapped into a "custom input stream". The purpose of this input stream is to close the response - // when the inputstream is itself closed. Closing the response is required to ensure that the resources are released - // back to the underlying pool. Otherwise, when doing a close(), the close() would stay hanging until the connection - // times out. - new HttpInputStream(response, request) - } catch { - case ex: Exception => - response.close() - ex match { - case ex: java.net.UnknownHostException => throw new HttpClientException(s"host not found for $url", ex) - case ex: IOException => throw new HttpClientException(s"unexpected error accessing $url", ex) - case _ => throw ex - } - } - } - - // This method is called by our 'http_request' intrinsic so does not check for the status code. - def getInputStreamWithStatus(url: String): HttpResult = { - val (response, request) = openHTTPConnection(url) - try { - val responseCode = response.getCode - - // In some cases, like a 204, there is no content, so creating an empty input-stream - val is = { - if (response.getEntity != null) new HttpInputStream(response, request) - else new ByteArrayInputStream(Array.empty) - } - val headers = response.getHeaders().map(x => x.getName -> x.getValue) - HttpResult(responseCode, is, headers) - } catch { - case ex: Exception => - response.close() - ex match { - case ex: java.net.UnknownHostException => throw new HttpClientException(s"host not found for $url", ex) - case ex: IOException => throw new HttpClientException(s"unexpected error accessing $url", ex) - case _ => throw ex - } - } - } - - override def getSeekableInputStream(url: String): SeekableInputStream = { - val skipableInputStream = new GenericSkippableInputStream(() => getInputStream(url)) - new DelegatingSeekableInputStream(skipableInputStream) { - override def getPos: Long = skipableInputStream.getPos - - override def seek(newPos: Long): Unit = skipableInputStream.seek(newPos) - } - } - - private def openHTTPConnection(url: String): (CloseableHttpResponse, HttpUriRequestBase) = { - val request = - try { - val uriBuilder = new URIBuilder(url) - args.foreach(x => uriBuilder.addParameter(x._1, x._2)) - val uri = uriBuilder.build() - method.toLowerCase match { - case "get" => new HttpGet(uri) - case "post" => new HttpPost(uri) - case "put" => new HttpPut(uri) - case "delete" => new HttpDelete(uri) - case "head" => new HttpHead(uri) - case "patch" => new HttpPatch(uri) - case "options" => new HttpOptions(uri) - } - } catch { - case ex: MalformedURLException => throw new LocationException(s"invalid HTTP URL: ${ex.getMessage}", ex) - case ex: URISyntaxException => throw new LocationException(s"invalid HTTP URL: ${ex.getMessage}", ex) - } - - headers.foreach { case (k, v) => request.setHeader(k, v) } - - body.foreach(s => - request match { - case _: HttpPost | _: HttpPut | _: HttpPatch => - request.setEntity(new ByteArrayEntity(s, ContentType.APPLICATION_OCTET_STREAM)) - case _ => throw new LocationException("only 'POST', 'PUT' and 'PATCH' HTTP requests can have a body") - } - ) - request.setConfig(requestConfig) - try { - (apacheHttpClient.httpClient.execute(request), request) - } catch { - case ex: java.net.UnknownHostException => throw new HttpClientException(s"host not found for $url", ex) - case ex: IOException => throw new HttpClientException(s"unexpected error accessing $url", ex) - } - } -} - -/** - * InputStream that closes response on close, to ensure resources are released even if responses not consumed. - * If we just closed the inputstream with is.close() - and not response.close() -, then it would hang until a timeout - * is reached. - */ -class HttpInputStream(response: CloseableHttpResponse, request: HttpUriRequestBase) - extends InputStream - with StrictLogging { - - private val is = response.getEntity.getContent - - override def read(b: Array[Byte]): Int = is.read(b) - - override def read(b: Array[Byte], off: Int, len: Int): Int = is.read(b, off, len) - - override def read(): Int = is.read() - - override def available(): Int = is.available() - - override def mark(readlimit: Int): Unit = is.mark(readlimit) - - override def markSupported(): Boolean = is.markSupported() - - override def reset(): Unit = is.reset() - - override def skip(n: Long): Long = is.skip(n) - - override def close(): Unit = { - // (CTM) response.close() was hanging until downloading all the file - // So following example here: - // https://github.com/apache/httpcomponents-client/blob/5.1.x/httpclient5/src/test/java/org/apache/hc/client5/http/examples/ClientAbortMethod.java - // calling response.close() after request.cancel() was throwing a SocketException with a message "Socket closed" - // So assuming the request.cancel also closes the response - request.cancel() - } -} diff --git a/sources/src/main/scala/raw/sources/bytestream/http/JavaRuntimeHttpClient.scala b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamClient.scala similarity index 70% rename from sources/src/main/scala/raw/sources/bytestream/http/JavaRuntimeHttpClient.scala rename to sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamClient.scala index bf89f5230..16f7801e4 100644 --- a/sources/src/main/scala/raw/sources/bytestream/http/JavaRuntimeHttpClient.scala +++ b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamClient.scala @@ -24,22 +24,24 @@ import java.net.http.HttpClient.Version import java.net.http.HttpRequest.{BodyPublisher, BodyPublishers} import java.net.http.HttpResponse.BodyHandlers import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.nio.charset.StandardCharsets import java.time.Duration import java.util.concurrent.TimeUnit import java.util.stream.Collectors import scala.collection.mutable -// TODO (msb): The existence of this is weird; also for the config settings it implies... -object JavaRuntimeHttpClient { +object HttpByteStreamClient { - import HttpClientSettings._ + private val HTTP_CONNECT_TIMEOUT = "raw.sources.bytestream.http.connect-timeout" + private val HTTP_READ_TIMEOUT = "raw.sources.bytestream.http.read-timeout" - // TODO: Transform this into dependency injection. Note that is requires 'settings'. - private val initLock = new Object + private val ERROR_RESPONSE_MAX_OUTPUT_SIZE = 2048 + + private val httpClientLock = new Object private var httpClient: HttpClient = _ - protected[http] def buildHttpClient(settings: RawSettings): HttpClient = { - initLock.synchronized { + def buildHttpClient(settings: RawSettings): HttpClient = { + httpClientLock.synchronized { if (httpClient == null) { val connectTimeout = settings.getDuration(HTTP_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) this.httpClient = java.net.http.HttpClient.newBuilder @@ -51,29 +53,44 @@ object JavaRuntimeHttpClient { } httpClient } + } -class JavaRuntimeHttpClient( - method: String, - args: Array[(String, String)], - headers: Array[(String, String)], - body: Option[Array[Byte]], - expectedStatus: Seq[Int] +class HttpByteStreamClient( + method: String = "GET", + args: Array[(String, String)] = Array.empty, + headers: Array[(String, String)] = Array.empty, + maybeBody: Option[Array[Byte]] = None, + expectedStatus: Array[Int] = Array( + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_ACCEPTED, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_PARTIAL + ) )(implicit settings: RawSettings) extends InputStreamClient - with RuntimeHttpClient with StrictLogging { - import HttpClientSettings._ + import HttpByteStreamClient._ - private val httpClient = JavaRuntimeHttpClient.buildHttpClient(settings) + private val httpClient = HttpByteStreamClient.buildHttpClient(settings) private val httpReadTimeoutMillis = settings.getDuration(HTTP_READ_TIMEOUT, TimeUnit.MILLISECONDS) private val requestBuilderTemplate = HttpRequest .newBuilder() .timeout(Duration.ofMillis(httpReadTimeoutMillis)) - // This method is called by our runtime in a read so expects the status code to be 200 + protected def readOutputBounded(is: InputStream): String = { + val data = is.readNBytes(ERROR_RESPONSE_MAX_OUTPUT_SIZE) + val result = new String(data, StandardCharsets.UTF_8) + if (data.length == ERROR_RESPONSE_MAX_OUTPUT_SIZE && is.read() != -1) { + result + "..." + } else { + result + } + } + + // This method expects the response status code to be 200. override def getInputStream(url: String): InputStream = { val response = openHTTPConnection(url) val responseCode = response.statusCode() @@ -89,16 +106,16 @@ class JavaRuntimeHttpClient( is.close() } if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw new HttpClientException(s"authorization error accessing $url: ($responseCode)\n$bodyContents") + throw new HttpByteStreamException(s"authorization error accessing $url: ($responseCode)\n$bodyContents") } else { - throw new HttpClientException( + throw new HttpByteStreamException( s"could not read (HTTP ${method.toUpperCase}) from $url: ($responseCode)\n$bodyContents" ) } } } - // This method is called by our 'http_request' intrinsic so does not check for the status code. + // This method does not check the response status code. def getInputStreamWithStatus(url: String): HttpResult = { val response = openHTTPConnection(url) val responseCode = response.statusCode() @@ -119,17 +136,17 @@ class JavaRuntimeHttpClient( HttpResult(responseCode, is, headersSeq) } - override def getSeekableInputStream(url: String): SeekableInputStream = { - val skipableInputStream = new GenericSkippableInputStream(() => getInputStream(url)) - new DelegatingSeekableInputStream(skipableInputStream) { - override def getPos: Long = skipableInputStream.getPos + def getSeekableInputStream(url: String): SeekableInputStream = { + val skippableInputStream = new GenericSkippableInputStream(() => getInputStream(url)) + new DelegatingSeekableInputStream(skippableInputStream) { + override def getPos: Long = skippableInputStream.getPos - override def seek(newPos: Long): Unit = skipableInputStream.seek(newPos) + override def seek(newPos: Long): Unit = skippableInputStream.seek(newPos) } } private def createBodyPublisher(): BodyPublisher = { - body + maybeBody .map(s => method.toLowerCase match { case "post" | "put" | "patch" => return BodyPublishers.ofByteArray(s) @@ -168,7 +185,7 @@ class JavaRuntimeHttpClient( case ex: IllegalArgumentException => // .setHeader docs says `IllegalArgumentException` can be thrown if the header is restricted (e.g. "Host") // RD-6871 - throw new HttpClientException(ex.getMessage, ex) + throw new HttpByteStreamException(ex.getMessage, ex) } } @@ -177,8 +194,8 @@ class JavaRuntimeHttpClient( try { httpClient.send(request, BodyHandlers.ofInputStream()) } catch { - case ex: java.net.ConnectException => throw new HttpClientException(s"host not found for $url", ex) - case ex: IOException => throw new HttpClientException(s"unexpected error accessing $url", ex) + case ex: java.net.ConnectException => throw new HttpByteStreamException(s"host not found for $url", ex) + case ex: IOException => throw new HttpByteStreamException(s"unexpected error accessing $url", ex) } } } diff --git a/sources/src/main/scala/raw/creds/oauth2/api/RenewedAccessToken.scala b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamException.scala similarity index 61% rename from sources/src/main/scala/raw/creds/oauth2/api/RenewedAccessToken.scala rename to sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamException.scala index 52403bf50..b287e4be6 100644 --- a/sources/src/main/scala/raw/creds/oauth2/api/RenewedAccessToken.scala +++ b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamException.scala @@ -10,14 +10,8 @@ * licenses/APL.txt. */ -package raw.creds.oauth2.api +package raw.sources.bytestream.http -import java.time.Instant +import raw.sources.bytestream.api.ByteStreamException -final case class RenewedAccessToken( - accessToken: String, - expiresBy: Instant, - scopes: Seq[String], - refreshToken: Option[String], - header: Option[String] = None -) +class HttpByteStreamException(message: String, cause: Throwable = null) extends ByteStreamException(message, cause) diff --git a/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocation.scala b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocation.scala index fa8595b85..b5f7fc866 100644 --- a/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocation.scala +++ b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocation.scala @@ -13,32 +13,47 @@ package raw.sources.bytestream.http import raw.sources.bytestream.api.{ByteStreamException, ByteStreamLocation, SeekableInputStream} -import raw.client.api.LocationDescription +import raw.utils.RawSettings import java.io.InputStream -import java.net.URI +import java.net.{HttpURLConnection, MalformedURLException, URI, URISyntaxException} import java.nio.file.Path final case class HttpResult(status: Int, is: InputStream, headers: Seq[(String, String)]) class HttpByteStreamLocation( - cli: RuntimeHttpClient, - url: String, - locationDescription: LocationDescription -) extends ByteStreamLocation { + val url: String, + val method: String = "GET", + val args: Array[(String, String)] = Array.empty, + val headers: Array[(String, String)] = Array.empty, + val maybeBody: Option[Array[Byte]] = None, + val expectedStatus: Array[Int] = Array( + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_ACCEPTED, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_PARTIAL + ) +)(implicit settings: RawSettings) + extends ByteStreamLocation { - override val rawUri: String = { - // normalizing the URL removes consecutive occurrences of /, which would make the Apache HTTP client fail. This is a - // convenience feature so that URLs like http://host:port//path will work correctly, mainly useful for URLs generated by - // the concatenation of several segments. + private val httpClient = + try { + new HttpByteStreamClient(method, args, headers, maybeBody, expectedStatus)( + settings + ) + } catch { + case ex: MalformedURLException => throw new HttpByteStreamException(s"invalid HTTP URL: ${ex.getMessage}", ex) + case ex: URISyntaxException => throw new HttpByteStreamException(s"invalid HTTP URL: ${ex.getMessage}", ex) + } + + private val safeUrl: String = { new URI(url).normalize().toString } override def testAccess(): Unit = { // We are reading a single byte to ensure the connection is valid. - // By reading a single byte, we are actually doing a connection and authentication - // this was done to fix a bug where http sources where not retrying - val is = cli.getInputStream(rawUri) + // By reading a single byte, we are actually doing a connection and authentication. + val is = httpClient.getInputStream(safeUrl) try { is.read() } finally { @@ -47,11 +62,11 @@ class HttpByteStreamLocation( } override protected def doGetInputStream(): InputStream = { - cli.getInputStream(rawUri) + httpClient.getInputStream(safeUrl) } override protected def doGetSeekableInputStream(): SeekableInputStream = { - cli.getSeekableInputStream(rawUri) + httpClient.getSeekableInputStream(safeUrl) } override def getLocalPath(): Path = { @@ -59,6 +74,7 @@ class HttpByteStreamLocation( } def getHttpResult(): HttpResult = { - cli.getInputStreamWithStatus(rawUri) + httpClient.getInputStreamWithStatus(safeUrl) } + } diff --git a/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocationBuilder.scala b/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocationBuilder.scala deleted file mode 100644 index 4a6d24d27..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/http/HttpByteStreamLocationBuilder.scala +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.http - -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.ObjectMapper -import com.typesafe.scalalogging.StrictLogging -import org.apache.commons.io.IOUtils -import org.apache.hc.client5.http.classic.methods.HttpPost -import org.apache.hc.core5.http.HttpHeaders -import org.apache.hc.core5.net.URIBuilder -import raw.client.api.LocationDescription -import raw.creds.api._ -import raw.creds.oauth2.auth0.Auth0OAuth2Client -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.bytestream.api.ByteStreamLocationBuilder -import raw.utils.RawSettings - -import java.io.IOException -import java.net.{HttpURLConnection, MalformedURLException, URISyntaxException} -import java.nio.charset.StandardCharsets -import java.util.Base64 -import scala.collection.mutable - -object HttpByteStreamLocationBuilder extends StrictLogging { - - private val mapper = new ObjectMapper() - - private def getClientCredsToken(clientId: String, clientSecret: String, server: String, useBasicAuth: Boolean)( - implicit settings: RawSettings - ): String = { - val response = - try { - // adding parameters - val uriBuilder = new URIBuilder(server) - if (useBasicAuth) { - uriBuilder.addParameter("grant_type", "client_credentials") - } else { - Seq("grant_type" -> "client_credentials", "client_id" -> clientId, "client_secret" -> clientSecret).foreach( - x => uriBuilder.addParameter(x._1, x._2) - ) - } - // Making the default a POST as most apis use that to renew the token. - // TODO: Add more options for the token req (method, json body) or a big if then else per API (twitter, RTS, facebook) - val request = new HttpPost(uriBuilder.build()) - if (useBasicAuth) { - val bytesEncoded = Base64.getEncoder.encode(s"$clientId:$clientSecret".getBytes) - request.setHeader(HttpHeaders.AUTHORIZATION, s"Basic ${new String(bytesEncoded)}") - } - request.setHeader(HttpHeaders.ACCEPT, "application/json") - request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache") - val apacheHttpClient = ApacheRuntimeHttpClient.buildApacheHttpClient(settings) - apacheHttpClient.httpClient.execute(request) - } catch { - case ex: MalformedURLException => throw new LocationException(s"invalid HTTP token URL '$server'", ex) - case ex: URISyntaxException => throw new LocationException(s"invalid HTTP token URL '$server'", ex) - case ex: java.net.UnknownHostException => throw new HttpClientException(s"host not found for $server", ex) - case ex: IOException => throw new HttpClientException(s"error obtaining token with HTTP endpoint $server", ex) - } - - try { - val respCode = response.getCode - if (response.getCode == 200) { - val is = response.getEntity.getContent - val contents = - try { - IOUtils.toString(is, StandardCharsets.UTF_8) - } finally { - is.close() - } - val jsonNode = mapper.readTree(contents) - val token = jsonNode.get("access_token") - if (token == null) { - throw new HttpClientException( - s"""error obtaining token with HTTP endpoint $server: received json does not have 'access_token' field. Response: "$contents" """ - ) - } - token.asText() - } else { - logger.warn(s"Error obtaining token for HTTP endpoint $server. Unexpected response code: $respCode") - throw new HttpClientException( - s"error obtaining token with HTTP endpoint $server, unexpected response code: $respCode", - null - ) - } - } catch { - case ex: IOException => throw new HttpClientException(s"error obtaining token with HTTP endpoint $server", ex) - case ex: JsonProcessingException => throw new HttpClientException( - s"error obtaining token with HTTP endpoint $server: error processing json response", - ex - ) - } finally { - response.close() - } - } - - private def getClientCredsTokenFromProvider( - clientId: String, - clientSecret: String, - provider: OAuth2Provider.Value, - options: Map[String, String] - )( - implicit settings: RawSettings - ): String = { - provider match { - case OAuth2Provider.Auth0 => - val client = new Auth0OAuth2Client() - val token = client.newAccessTokenFromClientCredentials(clientId, clientSecret, options) - token.accessToken - case _ => throw new HttpClientException( - s"provider ${provider.toString.toLowerCase} no supported for client-id/client-secret credentials" - ) - } - } - -} - -class HttpByteStreamLocationBuilder extends ByteStreamLocationBuilder with StrictLogging { - import HttpByteStreamLocationBuilder._ - - private val httpRegex = """(http|https)://(.*)""".r - - override def schemes: Seq[String] = Seq("http", "https") - - private def resolveCredentials( - location: LocationDescription - )(implicit sourceContext: SourceContext): Option[NewHttpAuth] = { - if (location.getStringSetting("http-auth-cred-name").isDefined) { - val authName = location.getStringSetting("http-auth-cred-name").get - val maybeCred = sourceContext.credentialsService.getNewHttpAuth(sourceContext.user, authName) - if (maybeCred.isEmpty) { - throw new HttpClientException(s"cannot find credential: $authName") - } - maybeCred - } else if (location.getStringSetting("http-token").isDefined) { - val token = location.getStringSetting("http-token").get - Some(BearerToken(token, Map.empty)) - } else if (location.getStringSetting("http-client-id").isDefined) { - val clientId = location.getStringSetting("http-client-id") - val clientSecret = location.getStringSetting("http-client-secret") - val authProvider = location.getStringSetting("http-auth-provider") - val tokenUrl = location.getStringSetting("http-token-url") - if (clientSecret.isEmpty) { - throw new LocationException( - "http client-id/client-secret credentials need http-client-id and http-client-secret" - ) - } - if (authProvider.isDefined) { - val options = location.getKVSetting("http-auth-options").getOrElse(Array.empty).toMap - val token = - getClientCredsTokenFromProvider(clientId.get, clientSecret.get, OAuth2Provider(authProvider.get), options)( - sourceContext.settings - ) - Some(BearerToken(token, Map.empty)) - } else if (tokenUrl.isDefined) { - val useBasicAuth = location.getBooleanSetting("http-use-basic-auth").getOrElse(false) - val token = - getClientCredsToken(clientId.get, clientSecret.get, tokenUrl.get, useBasicAuth)(sourceContext.settings) - Some(BearerToken(token, Map.empty)) - } else { - throw new LocationException( - "http client-id/secret credentials need one of: http-auth-provider or http-token-url property" - ) - } - - } else if (location.getStringSetting("http-user-name").isDefined) { - val userName = location.getStringSetting("http-user-name").get - val password = location - .getStringSetting("http-password") - .getOrElse( - throw new LocationException( - "http basic-auth credentials need http-username and http-password properties" - ) - ) - Some(BasicAuth(userName, password, Map.empty)) - - } else { - sourceContext.credentialsService - .getHTTPCred(sourceContext.user, location.url) - .map(c => - c.credentials match { - case OauthClientCredentials(clientId, clientSecret, tokenUrl, useBasicAuth) => - val token = getClientCredsToken(clientId, clientSecret, tokenUrl, useBasicAuth)(sourceContext.settings) - BearerToken(token, Map.empty) - case OauthToken(token, _, _) => BearerToken(token, Map.empty) - case BasicAuthCredentials(user, password) => BasicAuth(user, password, Map.empty) - case AccessToken(accessToken) => BearerToken(accessToken, Map.empty) - } - ) - } - } - - @throws(classOf[LocationException]) - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): HttpByteStreamLocation = { - location.url match { - case httpRegex(_, _) => - // Test if URL is valid - try { - // Build the location - val method = location.getStringSetting("http-method").getOrElse("get") - val args = location.getKVSetting("http-args").getOrElse(Array.empty) - val headers: mutable.Map[String, String] = location - .getKVSetting("http-headers") - .map(a => { - val mutableMap = new mutable.HashMap[String, String]() - a.foreach(e => mutableMap += e) - mutableMap - }) - .getOrElse(new mutable.HashMap[String, String]()) - val body = location.getBinarySetting("http-body") - val bodyString = location.getStringSetting("http-body-string").map(_.getBytes("utf-8")) - - if (!headers.contains(HttpHeaders.AUTHORIZATION)) { - resolveCredentials(location) match { - case Some(BearerToken(token, _)) => headers.put(HttpHeaders.AUTHORIZATION, s"Bearer $token") - case Some(BasicAuth(username, password, _)) => - val bytesEncoded = Base64.getEncoder.encode(s"$username:$password".getBytes) - headers.put(HttpHeaders.AUTHORIZATION, s"Basic ${new String(bytesEncoded)}") - case Some(CustomHeaderToken(header, token, _)) => headers.put(header, token) - case None => - } - } - - val expectedStatus = location - .getIntArraySetting("http-expected-status") - .getOrElse( - Array( - HttpURLConnection.HTTP_OK, - HttpURLConnection.HTTP_ACCEPTED, - HttpURLConnection.HTTP_CREATED, - HttpURLConnection.HTTP_PARTIAL - ) - ) - val cli = new JavaRuntimeHttpClient(method, args, headers.to, body.orElse(bodyString), expectedStatus)( - sourceContext.settings - ) - new HttpByteStreamLocation( - cli, - location.url, - location - ) - } catch { - case ex: MalformedURLException => throw new LocationException(s"invalid HTTP URL: ${ex.getMessage}", ex) - case ex: URISyntaxException => throw new LocationException(s"invalid HTTP URL: ${ex.getMessage}", ex) - } - case _ => throw new LocationException("not an HTTP location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/bytestream/http/RuntimeHttpClient.scala b/sources/src/main/scala/raw/sources/bytestream/http/RuntimeHttpClient.scala deleted file mode 100644 index 6f5b061bb..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/http/RuntimeHttpClient.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.http - -import raw.sources.bytestream.api.SeekableInputStream - -import java.io.InputStream -import java.nio.charset.StandardCharsets - -trait RuntimeHttpClient { - - // This method is called by our runtime in a read so expects the status code to be 200 - def getInputStream(url: String): InputStream - - // This method is called by our 'http_request' intrinsic so does not check for the status code. - def getInputStreamWithStatus(url: String): HttpResult - - def getSeekableInputStream(url: String): SeekableInputStream - - private val ErrorResponseOutputMaxSize = 2048 - - protected def readOutputBounded(is: InputStream): String = { - val data = is.readNBytes(ErrorResponseOutputMaxSize) - val result = new String(data, StandardCharsets.UTF_8) - if (data.length == ErrorResponseOutputMaxSize && is.read() != -1) { - result + "..." - } else { - result - } - } -} diff --git a/sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocationBuilder.scala b/sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocationBuilder.scala deleted file mode 100644 index cca7d4050..000000000 --- a/sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocationBuilder.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.in_memory - -import raw.sources.bytestream.api.{ByteStreamLocation, ByteStreamLocationBuilder} -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -class InMemoryByteStreamLocationBuilder extends ByteStreamLocationBuilder { - - override def schemes: Seq[String] = Seq(InMemoryByteStreamLocation.schema) - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): ByteStreamLocation = { - if (location.url.startsWith(schemes.head)) { - new InMemoryByteStreamLocation(location) - } else { - throw new LocationException(s"not an in-memory location") - } - - } - -} diff --git a/sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocation.scala b/sources/src/main/scala/raw/sources/bytestream/inmemory/InMemoryByteStreamLocation.scala similarity index 57% rename from sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocation.scala rename to sources/src/main/scala/raw/sources/bytestream/inmemory/InMemoryByteStreamLocation.scala index 181d0bd82..ea08c2c67 100644 --- a/sources/src/main/scala/raw/sources/bytestream/in_memory/InMemoryByteStreamLocation.scala +++ b/sources/src/main/scala/raw/sources/bytestream/inmemory/InMemoryByteStreamLocation.scala @@ -10,9 +10,9 @@ * licenses/APL.txt. */ -package raw.sources.bytestream.in_memory +package raw.sources.bytestream.inmemory -import raw.client.api._ +import raw.sources.api.LocationException import raw.sources.bytestream.api.{ ByteStreamLocation, DelegatingSeekableInputStream, @@ -23,23 +23,12 @@ import raw.sources.bytestream.api.{ import java.io.{ByteArrayInputStream, InputStream} import java.nio.file.Path -object InMemoryByteStreamLocation { - val schema = "in-memory" - val schemaWithColon = s"$schema:" - val codeDataKey = "code-data" -} +class InMemoryByteStreamLocation(val data: Array[Byte]) extends ByteStreamLocation { -class InMemoryByteStreamLocation( - locationDescription: LocationDescription -) extends ByteStreamLocation { + def this(data: String) = this(data.getBytes("UTF-8")) override protected def doGetInputStream(): InputStream = { - assert(locationDescription.settings.contains(LocationSettingKey(InMemoryByteStreamLocation.codeDataKey))) - - val LocationBinarySetting(v) = - locationDescription.settings(LocationSettingKey(InMemoryByteStreamLocation.codeDataKey)) - - new ByteArrayInputStream(v.toArray) + new ByteArrayInputStream(data) } override protected def doGetSeekableInputStream(): SeekableInputStream = { @@ -50,9 +39,8 @@ class InMemoryByteStreamLocation( } } - override def getLocalPath(): Path = throw new AssertionError("Calling path on in memory location") - - override def rawUri: String = InMemoryByteStreamLocation.schemaWithColon + override def getLocalPath(): Path = throw new LocationException("not supported for in-memory location") override def testAccess(): Unit = {} + } diff --git a/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationBuilder.scala b/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationBuilder.scala deleted file mode 100644 index 6f0962a47..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationBuilder.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.api - -import raw.client.api.LocationDescription -import raw.sources.api.SourceContext -import raw.sources.bytestream.api.ByteStreamLocationBuilder - -abstract class FileSystemLocationBuilder extends ByteStreamLocationBuilder { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation - -} diff --git a/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationProvider.scala b/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationProvider.scala deleted file mode 100644 index 012d32473..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/api/FileSystemLocationProvider.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.api - -import raw.utils.RawException - -import raw.sources.api.{LocationProvider, SourceContext} -import raw.client.api.LocationDescription - -object FileSystemLocationProvider extends LocationProvider { - - @throws[RawException] - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation = { - getScheme(location.url) match { - case Some(scheme) => - val impls = sourceContext.fileSystemLocationBuilderServices.filter(_.schemes.contains(scheme)) - if (impls.isEmpty) throw new FileSystemException(s"no file system location implementation found for $scheme") - else if (impls.length > 1) - throw new FileSystemException(s"more than one file system location implementation found for $scheme") - else impls.head.build(location) - case None => throw new FileSystemException(s"invalid url: '${location.url}'") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxPath.scala b/sources/src/main/scala/raw/sources/filesystem/dropbox/BaseDropboxPath.scala similarity index 62% rename from sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxPath.scala rename to sources/src/main/scala/raw/sources/filesystem/dropbox/BaseDropboxPath.scala index 5201293ce..5e999dcc1 100644 --- a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxPath.scala +++ b/sources/src/main/scala/raw/sources/filesystem/dropbox/BaseDropboxPath.scala @@ -12,20 +12,20 @@ package raw.sources.filesystem.dropbox -import raw.client.api._ +import com.dropbox.core.v2.DbxClientV2 import raw.sources.bytestream.api.{ByteStreamException, SeekableInputStream} import raw.sources.filesystem.api._ import java.io.InputStream import java.nio.file.Path -class DropboxPath( - cli: DropboxFileSystem, - path: String, - locationDescription: LocationDescription -) extends FileSystemLocation { +object BaseDropboxPath { + val DROPBOX_CLIENT_ID = "raw.sources.dropbox.clientId" +} + +abstract class BaseDropboxPath(dbxClientV2: DbxClientV2, path: String) extends FileSystemLocation { - override def rawUri: String = s"dropbox://${cli.name}$path" + protected val cli = new DropboxFileSystem(dbxClientV2) override def testAccess(): Unit = { cli.testAccess(path) @@ -47,16 +47,4 @@ class DropboxPath( cli.metadata(path) } - override protected def doLs(): Iterator[FileSystemLocation] = { - cli - .listContents(path) - .map(npath => new DropboxPath(cli, npath, locationDescription)) - } - - override protected def doLsWithMetadata(): Iterator[(FileSystemLocation, FileSystemMetadata)] = { - cli.listContentsWithMetadata(path).map { - case (npath, meta) => (new DropboxPath(cli, npath, locationDescription), meta) - } - } - } diff --git a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxAccessTokenPath.scala b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxAccessTokenPath.scala new file mode 100644 index 000000000..753fe3939 --- /dev/null +++ b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxAccessTokenPath.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.filesystem.dropbox + +import com.dropbox.core.DbxRequestConfig +import com.dropbox.core.oauth.DbxCredential +import com.dropbox.core.v2.DbxClientV2 +import raw.sources.filesystem.api.{FileSystemLocation, FileSystemMetadata} +import raw.utils.RawSettings + +class DropboxAccessTokenPath(val accessToken: String, val path: String, dbxClientV2: DbxClientV2) + extends BaseDropboxPath(dbxClientV2, path) { + + def this(accessToken: String, path: String)(implicit settings: RawSettings) = this( + accessToken, + path, + new DbxClientV2( + DbxRequestConfig.newBuilder(settings.getString(BaseDropboxPath.DROPBOX_CLIENT_ID)).build(), + new DbxCredential(accessToken) + ) + ) + + override protected def doLs(): Iterator[FileSystemLocation] = { + cli + .listContents(path) + .map(npath => new DropboxAccessTokenPath(accessToken, npath, dbxClientV2)) + } + + override protected def doLsWithMetadata(): Iterator[(FileSystemLocation, FileSystemMetadata)] = { + cli.listContentsWithMetadata(path).map { + case (npath, meta) => (new DropboxAccessTokenPath(accessToken, npath, dbxClientV2), meta) + } + } + +} diff --git a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystem.scala b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystem.scala index da198382f..98b58c864 100644 --- a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystem.scala +++ b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystem.scala @@ -13,47 +13,23 @@ package raw.sources.filesystem.dropbox import com.dropbox.core._ -import com.dropbox.core.oauth.DbxCredential import com.dropbox.core.v2.DbxClientV2 import com.dropbox.core.v2.files.{DownloadErrorException, FolderMetadata, Metadata, FileMetadata => DropboxFileMetadata} import org.springframework.util.AntPathMatcher -import raw.creds.api.{BasicAuth, BearerToken, NewHttpAuth} import raw.sources.bytestream.api.{DelegatingSeekableInputStream, GenericSkippableInputStream, SeekableInputStream} import raw.sources.filesystem.api._ -import raw.utils.RawSettings import java.io.InputStream import scala.collection.JavaConverters._ import scala.collection.mutable -object DropboxFileSystem { - private val DROPBOX_CLIENT_ID = "raw.sources.dropbox.clientId" - - // Method also used by testing infrastructure - private[sources] def buildDbxClientV2(cred: NewHttpAuth)(implicit settings: RawSettings): DbxClientV2 = { - val clientId = settings.getString(DROPBOX_CLIENT_ID) - val dropboxConfig = DbxRequestConfig.newBuilder(clientId).build() - val dbxCred = cred match { - case BasicAuth(user, password, _) => new DbxCredential(null, null, null, user, password) - case BearerToken(token, _) => new DbxCredential(token) - case _ => ??? - } - new DbxClientV2(dropboxConfig, dbxCred) - } -} +// TODO (msb): Catch unauthorized and throw specific exception? +class DropboxFileSystem(private[raw] val client: DbxClientV2) extends BaseFileSystem { -// TODO: Catch unauthorized to throw specific exception? -class DropboxFileSystem(cred: NewHttpAuth, val name: String = "")(implicit settings: RawSettings) - extends BaseFileSystem { + val fileSeparator: String = "/" - import DropboxFileSystem._ - - // TODO: this is arguable for Windows users... - private[sources] val fileSeparator: String = "/" private val fileSeparatorRegex: String = "/" - private lazy val client = buildDbxClientV2(cred) - private def sanitizePath(path: String): String = { val p = path.replaceAll(s"$fileSeparatorRegex+", fileSeparator).stripSuffix(fileSeparator) if (p.isEmpty) fileSeparator else p @@ -63,13 +39,8 @@ class DropboxFileSystem(cred: NewHttpAuth, val name: String = "")(implicit setti try { client.files().getMetadata(path) } catch { - case ex: DbxException => - logger.warn(s"Failed gracefully to get Dropbox path metadata: $path.", ex) - throwSourceException(ex, path) - case ex: IllegalArgumentException => - // Dropbox fails "other exceptions" as well when path isn't accessible/correct. - logger.warn(s"Failed unexpectedly to get Dropbox path metadata: $path.", ex) - throw new PathNotFoundException(path, ex) + case ex: DbxException => throwSourceException(ex, path) + case ex: IllegalArgumentException => throw new PathNotFoundException(path, ex) } } @@ -88,9 +59,7 @@ class DropboxFileSystem(cred: NewHttpAuth, val name: String = "")(implicit setti client.files().download(sanitizePath(file)).getInputStream } catch { // TODO: This misses NotAFileException I believe? - case ex: DbxException => - logger.warn(s"Failed to get Dropbox file inputstream: $file.", ex) - throwSourceException(ex, file) + case ex: DbxException => throwSourceException(ex, file) case ex: IllegalArgumentException => throw new PathInvalidException(file, ex) } @@ -151,9 +120,7 @@ class DropboxFileSystem(cred: NewHttpAuth, val name: String = "")(implicit setti if (recursive) client.files().listFolderBuilder(path).withRecursive(true).start() else client.files().listFolder(path) } catch { - case ex: DbxException => - logger.warn(s"Failed to list Dropbox folder: $path.", ex) - throwSourceException(ex, path) + case ex: DbxException => throwSourceException(ex, path) } while (continue) { lfr.getEntries.asScala.foreach { @@ -168,9 +135,7 @@ class DropboxFileSystem(cred: NewHttpAuth, val name: String = "")(implicit setti try { client.files().listFolderContinue(lfr.getCursor) } catch { - case ex: DbxException => - logger.warn(s"Failed to continue to list Dropbox folder: $path.", ex) - throwSourceException(ex, path) + case ex: DbxException => throwSourceException(ex, path) } } else { continue = false diff --git a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystemLocationBuilder.scala b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystemLocationBuilder.scala deleted file mode 100644 index 91a1a5c34..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxFileSystemLocationBuilder.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.dropbox - -import com.typesafe.scalalogging.StrictLogging -import raw.client.api.LocationDescription -import raw.creds.api.{BearerToken, DropboxToken} -import raw.sources.filesystem.api.{FileSystemException, FileSystemLocation, FileSystemLocationBuilder} -import raw.sources.api.SourceContext - -object DropboxFileSystemLocationBuilder { - val dropboxRegex = "dropbox:(?://([^/]+)?)?(.*)".r -} -class DropboxFileSystemLocationBuilder extends FileSystemLocationBuilder with StrictLogging { - - // Syntax: dropbox:/ - // user may be empty - override def schemes: Seq[String] = Seq("dropbox") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation = { - import DropboxFileSystemLocationBuilder._ - - location.url match { - case dropboxRegex(name, path) => - if (name == null) { - sourceContext.credentialsService.getDropboxToken(sourceContext.user) match { - case Some(cred: DropboxToken) => - val cli = new DropboxFileSystem(BearerToken(cred.accessToken, Map.empty))(sourceContext.settings) - new DropboxPath( - cli, - path, - location - ) - case _ => throw new FileSystemException("no credential found for Dropbox") - } - } else { - logger.debug(s"location: $location, name: $name, path: $path") - sourceContext.credentialsService.getNewHttpAuth(sourceContext.user, name) match { - case Some(cred: BearerToken) => - val cli = new DropboxFileSystem(cred, name)(sourceContext.settings) - new DropboxPath( - cli, - path, - location - ) - case Some(crds @ _) => - throw new FileSystemException(s"invalid credential type for Dropbox: ${crds.getClass}") - case _ => throw new FileSystemException("no credential found for Dropbox") - } - } - - case _ => throw new FileSystemException(s"not a Dropbox location: ${location.url}") - } - - } -} diff --git a/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxUsernamePasswordPath.scala b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxUsernamePasswordPath.scala new file mode 100644 index 000000000..88887f647 --- /dev/null +++ b/sources/src/main/scala/raw/sources/filesystem/dropbox/DropboxUsernamePasswordPath.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.filesystem.dropbox + +import com.dropbox.core.DbxRequestConfig +import com.dropbox.core.oauth.DbxCredential +import com.dropbox.core.v2.DbxClientV2 +import raw.sources.filesystem.api.{FileSystemLocation, FileSystemMetadata} +import raw.utils.RawSettings + +class DropboxUsernamePasswordPath( + val username: String, + val password: String, + val path: String, + dbxClientV2: DbxClientV2 +) extends BaseDropboxPath(dbxClientV2, path) { + + def this(username: String, password: String, path: String)( + implicit settings: RawSettings + ) = this( + username, + password, + path, + new DbxClientV2( + DbxRequestConfig.newBuilder(settings.getString(BaseDropboxPath.DROPBOX_CLIENT_ID)).build(), + new DbxCredential(null, null, null, username, password) + ) + ) + + override protected def doLs(): Iterator[FileSystemLocation] = { + cli + .listContents(path) + .map(npath => new DropboxUsernamePasswordPath(username, password, npath, dbxClientV2)) + } + + override protected def doLsWithMetadata(): Iterator[(FileSystemLocation, FileSystemMetadata)] = { + cli.listContentsWithMetadata(path).map { + case (npath, meta) => (new DropboxUsernamePasswordPath(username, password, npath, dbxClientV2), meta) + } + } + +} diff --git a/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystem.scala b/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystem.scala index 6d8ae5d33..69614a33e 100644 --- a/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystem.scala +++ b/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystem.scala @@ -22,15 +22,12 @@ import java.nio.file._ import java.nio.file.attribute.BasicFileAttributes import scala.collection.mutable.ArrayBuffer -class LocalFileSystem extends BaseFileSystem { +object LocalFileSystem extends BaseFileSystem { private[sources] val fileSeparator: String = File.separator private val fileSeparatorRegex: String = RawUtils.descape(fileSeparator) - // TODO (msb): This should fail to create if not on developer mode? - // TODO (msb): And if so, should require all paths to be under some certain base path? - private def sanitizePath(path: String): String = { // Currently there's no need to sanitize path because the Java APIs handle it for us. if (RawUtils.isWindows) path.replaceFirst("^/(.:/)", "$1") @@ -111,7 +108,6 @@ class LocalFileSystem extends BaseFileSystem { getPath(pathBeforeGlob), new SimpleFileVisitor[Path] { override def visitFileFailed(file: Path, exc: IOException): FileVisitResult = { - logger.trace(s"Cannot access path, skipping: $file: ${exc.toString}") FileVisitResult.SKIP_SUBTREE } override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { @@ -164,5 +160,3 @@ class LocalFileSystem extends BaseFileSystem { } } - -object LocalFileSystem extends LocalFileSystem diff --git a/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystemLocationBuilder.scala b/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystemLocationBuilder.scala deleted file mode 100644 index 125f12bd4..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/local/LocalFileSystemLocationBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.local - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.filesystem.api.{FileSystemLocation, FileSystemLocationBuilder} - -class LocalFileSystemLocationBuilder extends FileSystemLocationBuilder { - - override def schemes: Seq[String] = Seq("file") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation = { - val url = location.url - if (url.startsWith("file:")) { - val f = url.stripPrefix("file:") - if (f.nonEmpty) new LocalPath(url.stripPrefix("file:")) - else throw new LocationException("not a local location") - } else throw new LocationException("not a local location") - } - -} diff --git a/sources/src/main/scala/raw/sources/filesystem/local/LocalPath.scala b/sources/src/main/scala/raw/sources/filesystem/local/LocalPath.scala index 79733e995..cb5c96e57 100644 --- a/sources/src/main/scala/raw/sources/filesystem/local/LocalPath.scala +++ b/sources/src/main/scala/raw/sources/filesystem/local/LocalPath.scala @@ -17,13 +17,10 @@ import java.nio.file.{Path, Paths} import raw.sources.bytestream.api.SeekableInputStream import raw.sources.filesystem.api._ -// TODO (msb): Remove dependency on LocalFileSystem in the few places where it's still needed. -class LocalPath(pathName: String) extends FileSystemLocation { +class LocalPath(val pathName: String) extends FileSystemLocation { def this(path: Path) = this(path.toAbsolutePath.toString) - override def rawUri: String = s"file:$pathName" - protected def path: Path = Paths.get(pathName) override def testAccess(): Unit = { diff --git a/sources/src/main/scala/raw/sources/filesystem/mock/MockFileSystemLocationBuilder.scala b/sources/src/main/scala/raw/sources/filesystem/mock/MockFileSystemLocationBuilder.scala deleted file mode 100644 index 87a9e4f81..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/mock/MockFileSystemLocationBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.mock - -import com.typesafe.config.{ConfigException, ConfigFactory} -import com.typesafe.scalalogging.StrictLogging -import raw.sources.filesystem.api.{FileSystemLocation, FileSystemLocationBuilder, FileSystemLocationProvider} -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -class MockFileSystemLocationBuilder extends FileSystemLocationBuilder with StrictLogging { - - override def schemes: Seq[String] = Seq("mock") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation = { - val url = location.url - if (url.startsWith("mock:")) { - val f = url.stripPrefix("mock:") - if (f.nonEmpty) { - val collonIdx = f.indexOf(":") - if (collonIdx == -1) { - throw new LocationException(s"not a mock location: $url: could not find properties section.") - } - val propertiesString = f.substring(0, collonIdx) - val delegateUri = f.substring(collonIdx + 1) - logger.debug( - s"Creating mock filesystem with configuration: $propertiesString and delegate filesystem: $delegateUri" - ) - try { - val parser = ConfigFactory.parseString(propertiesString) - val delay = parser.getDuration("delay").toMillis - val delegate: FileSystemLocation = FileSystemLocationProvider.build( - LocationDescription(delegateUri, location.settings) - ) - new MockPath(delay, delegate) - } catch { - case ex: ConfigException => throw new LocationException(s"not a mock location: $url", ex) - } - } else { - throw new LocationException(s"not a mock location: $url") - } - } else throw new LocationException(s"not a mock location: $url") - } -} diff --git a/sources/src/main/scala/raw/sources/filesystem/mock/MockPath.scala b/sources/src/main/scala/raw/sources/filesystem/mock/MockPath.scala index 3244d8970..99d290c54 100644 --- a/sources/src/main/scala/raw/sources/filesystem/mock/MockPath.scala +++ b/sources/src/main/scala/raw/sources/filesystem/mock/MockPath.scala @@ -20,11 +20,7 @@ import java.io.InputStream import java.lang.StackWalker.StackFrame import java.nio.file.Path -class MockPath( - delayMillis: Long, - delegate: FileSystemLocation -) extends FileSystemLocation - with StrictLogging { +class MockPath(val delayMillis: Long, val delegate: FileSystemLocation) extends FileSystemLocation with StrictLogging { private def doDelay(): Unit = { val sw = StackWalker.getInstance() @@ -37,8 +33,6 @@ class MockPath( logger.info(s"Continuing") } - override def rawUri: String = s"mock:${delegate.rawUri}" - override def testAccess(): Unit = { doDelay() delegate.testAccess() diff --git a/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystem.scala b/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystem.scala index 28663f223..35b5a4f98 100644 --- a/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystem.scala +++ b/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystem.scala @@ -19,7 +19,6 @@ import software.amazon.awssdk.services.s3.S3Configuration import java.net.ConnectException import com.google.common.collect.AbstractIterator import org.springframework.util.AntPathMatcher -import raw.creds.api.{AWSCredentials, S3Bucket} import raw.sources.bytestream.api.{DelegatingSeekableInputStream, GenericSkippableInputStream, SeekableInputStream} import raw.sources.filesystem.api._ import raw.utils._ @@ -46,45 +45,45 @@ import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ object S3FileSystem { - private val S3_CONNECT_TIMEOUT = "raw.sources.s3.connect-timeout" - private val S3_READ_TIMEOUT = "raw.sources.s3.read-timeout" - private val S3_MAX_CONNECTIONS = "raw.sources.s3.max-connections" - private val S3_DEFAULT_REGION = "raw.sources.s3.default-region" + private val CONNECT_TIMEOUT = "raw.sources.s3.connect-timeout" + private val READ_TIMEOUT = "raw.sources.s3.read-timeout" + private val MAX_CONNECTIONS = "raw.sources.s3.max-connections" + private val DEFAULT_REGION = "raw.sources.s3.default-region" } -class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends BaseFileSystem { +class S3FileSystem( + val bucket: String, + val maybeRegion: Option[String], + val maybeAccessKey: Option[String], + val maybeSecretKey: Option[String] +)(implicit settings: RawSettings) + extends BaseFileSystem { import S3FileSystem._ - private lazy val defaultRegion = settings.getString(S3_DEFAULT_REGION) + private lazy val defaultRegion = settings.getString(DEFAULT_REGION) private[sources] val fileSeparator: String = "/" private val fileSeparatorRegex: String = RawUtils.descape(fileSeparator) - private val s3ConnectTimeout = settings.getDuration(S3_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) - private val s3ReadTimeout = settings.getDuration(S3_READ_TIMEOUT, TimeUnit.MILLISECONDS) - private val s3MaxConnections = settings.getInt(S3_MAX_CONNECTIONS) + private val s3ConnectTimeout = settings.getDuration(CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) + private val s3ReadTimeout = settings.getDuration(READ_TIMEOUT, TimeUnit.MILLISECONDS) + private val s3MaxConnections = settings.getInt(MAX_CONNECTIONS) - def bucketName: String = bucket.name - - private def guessBucketRegion(bucketName: String): Region = { + private def guessBucketRegion(): Region = { // Try to guess the bucket. // That said, we can only get the bucket if we have credentials. // If we don't have credentials, we also don't have permissions to find the bucket region. - if (bucket.credentials.isDefined) { + if (maybeAccessKey.isDefined && maybeSecretKey.isDefined) { val builder = S3Client.builder() builder.region(Region.of(defaultRegion)) - bucket.credentials match { - case Some(AWSCredentials(accessKey, secretKey)) => builder.credentialsProvider( - StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)) - ) - case None => builder.credentialsProvider(AnonymousCredentialsProvider.create()) - } + builder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(maybeAccessKey.get, maybeSecretKey.get)) + ) val client = builder.build() try { - val location = client.getBucketLocation(GetBucketLocationRequest.builder().bucket(bucketName).build()) + val location = client.getBucketLocation(GetBucketLocationRequest.builder().bucket(bucket).build()) val bucketRegion = location.locationConstraint() - // 'US_EAST_1' is returned as "null" by AWS SDK if (bucketRegion == null || bucketRegion.toString == "null" || bucketRegion.toString.isEmpty) Region.US_EAST_1 else Region.of(bucketRegion.toString) @@ -107,17 +106,19 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends builder.serviceConfiguration(s3Config) // Setting the region - val region = bucket.region match { + val region = maybeRegion match { case Some(regionValue) => Region.of(regionValue) - case None => guessBucketRegion(bucket.name) + case None => guessBucketRegion() } builder.region(region) // Set credentials - bucket.credentials match { - case Some(AWSCredentials(accessKey, secretKey)) => - builder.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey))) - case None => builder.credentialsProvider(AnonymousCredentialsProvider.create()) + if (maybeAccessKey.isDefined && maybeSecretKey.isDefined) { + builder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(maybeAccessKey.get, maybeSecretKey.get)) + ) + } else { + builder.credentialsProvider(AnonymousCredentialsProvider.create()) } val httpClient: SdkHttpClient = ApacheHttpClient @@ -159,7 +160,7 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends private def getS3Object(file: String) = { val getObjectRequest = GetObjectRequest .builder() - .bucket(bucket.name) + .bucket(bucket) .key(file) .build() @@ -208,7 +209,7 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends override def getInputStream(file: String): InputStream = { try { - new RawS3InputStream(getS3Object(sanitizePath(file))) + new S3InputStream(getS3Object(sanitizePath(file))) } catch { case ex: PathNotFoundException => // If there are contents, then try as "directory", which may throw exception @@ -257,7 +258,7 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends private var objectListing: ListObjectsV2Response = { val s3Request = ListObjectsV2Request .builder() - .bucket(bucket.name) + .bucket(bucket) .prefix(pathBeforeGlob) .build() @@ -314,7 +315,7 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends if (objectListing.isTruncated) { val nextRequest = ListObjectsV2Request .builder() - .bucket(bucket.name) + .bucket(bucket) .prefix(pathBeforeGlob) .continuationToken(objectListing.nextContinuationToken()) .build() @@ -396,16 +397,15 @@ class S3FileSystem(val bucket: S3Bucket)(implicit settings: RawSettings) extends } } - @throws[FileSystemException] def testBucketAccess(): Unit = { try { - client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket.name).build()) + client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) } catch { - case ex: SdkServiceException if ex.statusCode() == 403 => throw new PathUnauthorizedException(bucket.name, ex) + case ex: SdkServiceException if ex.statusCode() == 403 => throw new PathUnauthorizedException(bucket, ex) - case ex: SdkServiceException => throw new PathNotFoundException(bucket.name, ex) + case ex: SdkServiceException => throw new PathNotFoundException(bucket, ex) - case ex: SdkClientException => throw new FileSystemUnavailableException(s"s3://${bucket.name}", ex) + case ex: SdkClientException => throw new FileSystemUnavailableException(s"s3://$bucket", ex) } } diff --git a/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystemLocationBuilder.scala b/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystemLocationBuilder.scala deleted file mode 100644 index 1e0c71b19..000000000 --- a/sources/src/main/scala/raw/sources/filesystem/s3/S3FileSystemLocationBuilder.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.s3 - -import com.typesafe.scalalogging.StrictLogging -import raw.creds.api.{AWSCredentials, S3Bucket} -import raw.sources.filesystem.api.{FileSystemLocation, FileSystemLocationBuilder} -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -class S3FileSystemLocationBuilder extends FileSystemLocationBuilder with StrictLogging { - - // TODO (msb): Maybe we should have it support 's3a' as well? - private val s3Regex = """s3://([a-z\d][-a-z\d.]*)(/.*)?""".r - - override def schemes: Seq[String] = Seq("s3") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): FileSystemLocation = { - location.url match { - case s3Regex(bucket, key) => - val nonNullKey = - if (key == null) { - "" - } else { - key - } - - val region = location.getStringSetting("s3-region") - val credentials = for { - accessKey <- location.getStringSetting("s3-access-key") - secretKey <- location.getStringSetting("s3-secret-key") - } yield AWSCredentials(accessKey, secretKey) - // If credentials are not provided in the code, we try to get them from the credentials service - val s3Bucket = credentials match { - case Some(cred) => S3Bucket(bucket, region, Some(cred)) - case None => sourceContext.credentialsService - .getS3Bucket(sourceContext.user, bucket) - .getOrElse( - S3Bucket(bucket, region, None) - ) - } - - val cli = new S3FileSystem(s3Bucket)( - sourceContext.settings - ) - new S3Path( - cli, - nonNullKey, - location - ) - case _ => throw new LocationException(s"not an S3 location") - } - - } - -} diff --git a/sources/src/main/scala/raw/sources/filesystem/s3/RawS3InputStream.scala b/sources/src/main/scala/raw/sources/filesystem/s3/S3InputStream.scala similarity index 51% rename from sources/src/main/scala/raw/sources/filesystem/s3/RawS3InputStream.scala rename to sources/src/main/scala/raw/sources/filesystem/s3/S3InputStream.scala index f6b293585..55f9b5469 100644 --- a/sources/src/main/scala/raw/sources/filesystem/s3/RawS3InputStream.scala +++ b/sources/src/main/scala/raw/sources/filesystem/s3/S3InputStream.scala @@ -19,12 +19,31 @@ import java.io.IOException import java.io.InputStream import java.io.InterruptedIOException -class RawS3InputStream(s3ObjectInputStream: ResponseInputStream[GetObjectResponse]) extends InputStream { +class S3InputStream(s3ObjectInputStream: ResponseInputStream[GetObjectResponse]) extends InputStream { @throws[IOException] - override def read(): Int = { + override def read(): Int = withInterruptedCheck(s3ObjectInputStream.read()) + + @throws[IOException] + override def read(b: Array[Byte]): Int = withInterruptedCheck(s3ObjectInputStream.read(b)) + + @throws[IOException] + override def read(b: Array[Byte], off: Int, len: Int): Int = + withInterruptedCheck(s3ObjectInputStream.read(b, off, len)) + + @throws[IOException] + override def readAllBytes(): Array[Byte] = withInterruptedCheck(s3ObjectInputStream.readAllBytes()) + + @throws[IOException] + override def readNBytes(b: Array[Byte], off: Int, len: Int): Int = + withInterruptedCheck(s3ObjectInputStream.readNBytes(b, off, len)) + + @throws[IOException] + override def readNBytes(len: Int): Array[Byte] = withInterruptedCheck(s3ObjectInputStream.readNBytes(len)) + + private def withInterruptedCheck[T](f: => T): T = { try { - s3ObjectInputStream.read() + f } catch { case ex: InterruptedException => // InterruptedException is thrown when a thread is interrupted during a blocking IO operation. @@ -33,4 +52,5 @@ class RawS3InputStream(s3ObjectInputStream: ResponseInputStream[GetObjectRespons throw new InterruptedIOException(ex.getMessage) } } + } diff --git a/sources/src/main/scala/raw/sources/filesystem/s3/S3Path.scala b/sources/src/main/scala/raw/sources/filesystem/s3/S3Path.scala index 1bb523d80..7f25b0b26 100644 --- a/sources/src/main/scala/raw/sources/filesystem/s3/S3Path.scala +++ b/sources/src/main/scala/raw/sources/filesystem/s3/S3Path.scala @@ -12,28 +12,34 @@ package raw.sources.filesystem.s3 -import com.typesafe.scalalogging.StrictLogging -import raw.creds.api.S3Bucket import raw.sources.bytestream.api.{ByteStreamException, SeekableInputStream} -import raw.sources.filesystem.api._ -import raw.client.api.LocationDescription +import raw.sources.filesystem.api.{FileSystemLocation, FileSystemMetadata} +import raw.utils.RawSettings import java.io.InputStream import java.nio.file.Path -class S3Path( - cli: S3FileSystem, - protected val path: String, - locationDescription: LocationDescription -) extends FileSystemLocation - with StrictLogging { +class S3Path private (cli: S3FileSystem, val path: String)(implicit settings: RawSettings) extends FileSystemLocation { - val bucket: S3Bucket = cli.bucket + val bucket: String = cli.bucket - // TODO (msb): Shouldn't we sanitize path? - override val rawUri: String = { - val sep = if (path.startsWith("/")) "" else "/" - s"s3://${cli.bucketName}$sep$path" + val region: Option[String] = cli.maybeRegion + + val maybeAccessKey: Option[String] = cli.maybeAccessKey + + val maybeSecretKey: Option[String] = cli.maybeSecretKey + + def this( + bucket: String, + maybeRegion: Option[String], + maybeAccessKey: Option[String], + maybeSecretKey: Option[String], + path: String + )(implicit settings: RawSettings) = { + this( + new S3FileSystem(bucket, maybeRegion, maybeAccessKey, maybeSecretKey), + path + ) } override def testAccess(): Unit = { @@ -59,11 +65,11 @@ class S3Path( override protected def doLs(): Iterator[FileSystemLocation] = { cli .listContents(path) - .map(npath => new S3Path(cli, npath, locationDescription)) + .map(npath => new S3Path(cli, npath)) } override protected def doLsWithMetadata(): Iterator[(FileSystemLocation, FileSystemMetadata)] = { - cli.listContentsWithMetadata(path).map { case (npath, meta) => (new S3Path(cli, npath, locationDescription), meta) } + cli.listContentsWithMetadata(path).map { case (npath, meta) => (new S3Path(cli, npath), meta) } } } diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcClient.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcClient.scala index f4f41826a..bbdf32a59 100644 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/api/JdbcClient.scala @@ -54,12 +54,14 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin import JdbcClient._ + DriverManager.setLoginTimeout(getLoginTimeout(TimeUnit.SECONDS).toInt) + def hostname: String def vendor: String // Database is optional because some databases do not have the concept of database (Teradata and Sqlite). - def database: Option[String] + def maybeDatabase: Option[String] // Wrap vendor-specific calls and ensure only RelationalDatabaseException is thrown. def wrapSQLException[T](f: => T): T @@ -75,17 +77,15 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin // new PoolingDataSource(connectionPool) // } - def username: Option[String] + def maybeUsername: Option[String] - def password: Option[String] - - DriverManager.setLoginTimeout(getLoginTimeout(TimeUnit.SECONDS).toInt) + def maybePassword: Option[String] def getConnection: Connection = { // For connection pool: // wrapSQLException(datasource.getConnection()) wrapSQLException { - val conn = DriverManager.getConnection(connectionString, username.orNull, password.orNull) + val conn = DriverManager.getConnection(connectionString, maybeUsername.orNull, maybePassword.orNull) conn.setNetworkTimeout(Executors.newSingleThreadExecutor(), getNetworkTimeout(TimeUnit.MILLISECONDS).toInt) conn } @@ -104,8 +104,8 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin listTables(schema).close() } - def testAccess(database: Option[String], maybeSchema: Option[String], table: String): Unit = { - tableMetadata(database, maybeSchema, table) + def testAccess(maybeSchema: Option[String], table: String): Unit = { + tableMetadata(maybeSchema, table) } def listSchemas: Iterator[String] with Closeable = { @@ -124,10 +124,10 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin SchemaMetadata() } - def tableMetadata(database: Option[String], maybeSchema: Option[String], table: String): TableMetadata = { + def tableMetadata(maybeSchema: Option[String], table: String): TableMetadata = { val conn = getConnection try { - val res = getTableMetadata(conn, database, maybeSchema, table) + val res = getTableMetadata(conn, maybeDatabase, maybeSchema, table) try { getTableTypeFromTableMetadata(res) } finally { @@ -138,7 +138,7 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin } } - private def getTableMetadata( + protected def getTableMetadata( conn: Connection, maybeDatabase: Option[String], maybeSchema: Option[String], @@ -155,11 +155,9 @@ abstract class JdbcClient()(implicit settings: RawSettings) extends StrictLoggin } } - /** - * Infer schema from table. - * Skip silently fields we do not understand (except if can't understand any field, in which case, fire an error.) - */ - private def getTableTypeFromTableMetadata(res: ResultSet): TableMetadata = { + // Infer schema from table. + // Skip silently fields we do not understand (except if can't understand any field, in which case, fire an error.) + protected def getTableTypeFromTableMetadata(res: ResultSet): TableMetadata = { val columns = mutable.ListBuffer[TableColumn]() while (wrapSQLException(res.next)) { val columnName = wrapSQLException(res.getString("COLUMN_NAME")) diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationBuilder.scala deleted file mode 100644 index 1659b79bf..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationBuilder, SourceContext} - -trait JdbcLocationBuilder extends LocationBuilder { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationProvider.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationProvider.scala deleted file mode 100644 index 4722f4da1..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocationProvider.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.sources.api.{LocationProvider, SourceContext} -import raw.client.api.LocationDescription - -object JdbcLocationProvider extends LocationProvider { - - @throws[JdbcLocationException] - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - getScheme(location.url) match { - case Some(scheme) => - val impls = sourceContext.jdbcLocationBuilderServices.filter(_.schemes.contains(scheme)) - if (impls.isEmpty) - throw new JdbcLocationException(s"no relational database location implementation found for $scheme") - else if (impls.size > 1) throw new JdbcLocationException( - s"more than one relational database location implementation found for $scheme" - ) - else impls.head.build(location) - case None => throw new JdbcLocationException(s"invalid url: '${location.url}'") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocation.scala index 4ce757ded..0e81ddf69 100644 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocation.scala @@ -16,9 +16,12 @@ import raw.sources.api.Location import java.io.Closeable -abstract class JdbcSchemaLocation(val jdbcClient: JdbcClient, val maybeSchema: Option[String]) extends Location { +abstract class JdbcSchemaLocation( + val jdbcClient: JdbcClient, + maybeSchema: Option[String] +) extends Location { - def listTables(): Iterator[String] with Closeable + def listTables(): Iterator[JdbcTableLocation] with Closeable final override def testAccess(): Unit = { maybeSchema match { diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationBuilder.scala deleted file mode 100644 index 959444046..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationBuilder, SourceContext} - -trait JdbcSchemaLocationBuilder extends LocationBuilder { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationProvider.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationProvider.scala deleted file mode 100644 index f8e17f709..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcSchemaLocationProvider.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.sources.api.{LocationProvider, SourceContext} -import raw.client.api.LocationDescription - -object JdbcSchemaLocationProvider extends LocationProvider { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - getScheme(location.url) match { - case Some(scheme) => - val impls = sourceContext.jdbcSchemaLocationBuilderServices.filter(_.schemes.contains(scheme)) - if (impls.isEmpty) throw new JdbcLocationException(s"no schema location implementation found for $scheme") - else if (impls.size > 1) - throw new JdbcLocationException(s"more than one schema location implementation found for $scheme") - else impls.head.build(location) - case None => throw new JdbcLocationException(s"invalid url: '${location.url}'") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocation.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcServerLocation.scala similarity index 72% rename from sources/src/main/scala/raw/sources/jdbc/api/JdbcLocation.scala rename to sources/src/main/scala/raw/sources/jdbc/api/JdbcServerLocation.scala index 1606665a6..1c96eadee 100644 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/api/JdbcServerLocation.scala @@ -13,16 +13,13 @@ package raw.sources.jdbc.api import java.sql.Connection -import com.typesafe.scalalogging.StrictLogging import raw.sources.api.Location import java.io.Closeable -abstract class JdbcLocation(val jdbcClient: JdbcClient, val vendor: String, val dbName: String) - extends Location - with StrictLogging { +abstract class JdbcServerLocation(val jdbcClient: JdbcClient) extends Location { - def listSchemas(): Iterator[String] with Closeable + def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable final def getJdbcConnection(): Connection = { jdbcClient.getConnection diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocation.scala index 412605355..7097a96be 100644 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocation.scala @@ -12,24 +12,20 @@ package raw.sources.jdbc.api -import com.typesafe.scalalogging.StrictLogging import raw.sources.api.Location abstract class JdbcTableLocation( val jdbcClient: JdbcClient, - val vendor: String, - val dbName: String, - val table: String, - val maybeSchema: Option[String] -) extends Location - with StrictLogging { + maybeSchema: Option[String], + table: String +) extends Location { final override def testAccess(): Unit = { - jdbcClient.testAccess(Some(dbName), maybeSchema, table) + jdbcClient.testAccess(maybeSchema, table) } final def getType(): TableMetadata = { - jdbcClient.tableMetadata(Some(dbName), maybeSchema, table) + jdbcClient.tableMetadata(maybeSchema, table) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationBuilder.scala deleted file mode 100644 index 9ccec6889..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationBuilder.scala +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationBuilder, SourceContext} - -trait JdbcTableLocationBuilder extends LocationBuilder { - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationProvider.scala b/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationProvider.scala deleted file mode 100644 index 671798c09..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/api/JdbcTableLocationProvider.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.api - -import raw.client.api.LocationDescription - -import raw.sources.api.{LocationProvider, SourceContext} - -object JdbcTableLocationProvider extends LocationProvider { - - def isSupported(url: String)(implicit sourceContext: SourceContext): Boolean = { - getScheme(url) match { - case Some(scheme) => sourceContext.jdbcTableLocationBuilderServices.exists(_.schemes.contains(scheme)) - case None => false - } - } - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - getScheme(location.url) match { - case Some(scheme) => - val impls = sourceContext.jdbcTableLocationBuilderServices.filter(_.schemes.contains(scheme)) - if (impls.isEmpty) throw new JdbcLocationException(s"no table location implementation found for $scheme") - else if (impls.size > 1) - throw new JdbcLocationException(s"more than one table location implementation found for $scheme") - else impls.head.build(location) - case None => throw new JdbcLocationException(s"invalid url: '${location.url}'") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClient.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClient.scala index 97ec9a91c..dab2f585b 100644 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClient.scala @@ -15,7 +15,6 @@ package raw.sources.jdbc.mysql import com.mysql.cj.exceptions.CJCommunicationsException import java.util.concurrent.TimeUnit -import raw.creds.api.MySqlCredential import raw.sources.jdbc.api._ import raw.utils.RawSettings @@ -23,7 +22,9 @@ import java.net.{SocketTimeoutException, UnknownHostException} import java.sql.SQLException import scala.util.control.NonFatal -class MySqlClient(db: MySqlCredential)(implicit settings: RawSettings) extends JdbcClient { +class MySqlClient(val hostname: String, val port: Int, dbName: String, username: String, password: String)( + implicit settings: RawSettings +) extends JdbcClient { Class.forName("com.mysql.cj.jdbc.Driver") @@ -31,16 +32,16 @@ class MySqlClient(db: MySqlCredential)(implicit settings: RawSettings) extends J private val readTimeout = getReadTimeout(TimeUnit.MILLISECONDS) override val vendor: String = "mysql" - override val connectionString: String = { - val maybePort = db.port.map(p => ":" + p).getOrElse("") - s"jdbc:$vendor://${db.host}$maybePort/${db.database}?connectTimeout=$connectTimeout&socketTimeout=$readTimeout" - } - override val username: Option[String] = db.username - override val password: Option[String] = db.password - override val hostname: String = db.host + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) - override val database: Option[String] = Some(db.database) + override val maybePassword: Option[String] = Some(password) + + override val connectionString: String = { + s"jdbc:$vendor://$hostname:$port/$dbName?connectTimeout=$connectTimeout&socketTimeout=$readTimeout" + } override def wrapSQLException[T](f: => T): T = { try { @@ -49,7 +50,7 @@ class MySqlClient(db: MySqlCredential)(implicit settings: RawSettings) extends J case ex: SQLException => ex.getCause match { case _: UnknownHostException => throw new RDBMSUnknownHostException(hostname, ex) case _: SocketTimeoutException => throw new RDBMSConnectTimeoutException(hostname, ex) - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // Some more codes here (DB2 Universal Messages manual), various databases have varying degrees of compliance //https://www.ibm.com/support/knowledgecenter/en/SS6NHC/com.ibm.swg.im.dashdb.messages.doc/doc/rdb2stt.html @@ -76,9 +77,8 @@ class MySqlClient(db: MySqlCredential)(implicit settings: RawSettings) extends J } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClients.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClients.scala deleted file mode 100644 index 1cafc266a..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlClients.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import raw.creds.api.MySqlCredential -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -object MySqlClients { - - def get(dbName: String, location: LocationDescription)(implicit sourceContext: SourceContext): MySqlClient = { - val cred: MySqlCredential = location.getStringSetting("db-host") match { - case Some(host) => - val port = location.getIntSetting("db-port") - val userName = location.getStringSetting("db-username") - val password = location.getStringSetting("db-password") - MySqlCredential(host, port, dbName, userName, password) - case _ => sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: MySqlCredential) => cred - case _ => throw new LocationException(s"no credential found for mysql: $dbName") - } - } - new MySqlClient(cred)(sourceContext.settings) - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocationBuilder.scala deleted file mode 100644 index 3640b2970..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class MySqlLocationBuilder extends JdbcLocationBuilder { - - private val mysqlTableRegex = """mysql:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("mysql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case mysqlTableRegex(dbName) => - val db = MySqlClients.get(dbName, location) - new MySqlLocation(db, db.database.get) - case _ => throw new LocationException("not a mysql database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchema.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchema.scala deleted file mode 100644 index ef9e4e4bb..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchema.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -class MySqlSchema( - cli: MySqlClient, - dbName: String -) extends JdbcSchemaLocation(cli, None) { - - override def rawUri: String = s"mysql:$dbName/" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables("") - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"mysql:$dbName/${it.next()}" - - override def close(): Unit = it.close() - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocation.scala new file mode 100644 index 000000000..9bc34ea23 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocation.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.mysql + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class MySqlSchemaLocation private (cli: MySqlClient) extends JdbcSchemaLocation(cli, None) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this(host: String, port: Int, dbName: String, username: String, password: String)( + implicit settings: RawSettings + ) = { + this(new MySqlClient(host, port, dbName, username, password)) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables("") // Schema is ignored for MySQL. + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new MySqlTableLocation(cli, it.next()) + } + + override def close(): Unit = it.close() + } + } +} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocationBuilder.scala deleted file mode 100644 index a6a064a73..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlSchemaLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class MySqlSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - - private val schemaRegex = """mysql:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("mysql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName) => - val db = MySqlClients.get(dbName, location) - new MySqlSchema(db, db.database.get) - case _ => throw new LocationException("not a mysql schema location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocation.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlServerLocation.scala similarity index 58% rename from sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocation.scala rename to sources/src/main/scala/raw/sources/jdbc/mysql/MySqlServerLocation.scala index c1cd0a81b..7e0340295 100644 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlServerLocation.scala @@ -14,15 +14,20 @@ package raw.sources.jdbc.mysql import java.io.Closeable import raw.sources.jdbc.api._ +import raw.utils.RawSettings -class MySqlLocation( - cli: MySqlClient, - dbName: String -) extends JdbcLocation(cli, "mysql", dbName) { +class MySqlServerLocation( + val host: String, + val port: Int, + val dbName: String, + val username: String, + val password: String +)( + implicit settings: RawSettings +) extends JdbcServerLocation(new MySqlClient(host, port, dbName, username, password)) { - override def rawUri: String = s"mysql:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { throw new JdbcLocationException("no schemas in mysql") } + } diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTable.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTable.scala deleted file mode 100644 index 1d4cfcf73..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTable.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import raw.sources.jdbc.api.JdbcTableLocation - -class MySqlTable( - cli: MySqlClient, - dbName: String, - table: String -) extends JdbcTableLocation(cli, "mysql", dbName, table, None) { - - override def rawUri: String = s"mysql:$dbName/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocation.scala new file mode 100644 index 000000000..02743af91 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocation.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.mysql + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class MySqlTableLocation(cli: MySqlClient, val table: String) extends JdbcTableLocation(cli, None, table) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this(host: String, port: Int, dbName: String, username: String, password: String, tableName: String)( + implicit settings: RawSettings + ) = { + this( + new MySqlClient(host, port, dbName, username, password), + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocationBuilder.scala deleted file mode 100644 index 4f44015fb..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/mysql/MySqlTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.mysql - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class MySqlTableLocationBuilder extends JdbcTableLocationBuilder { - - private val mysqlTableRegex = """mysql:(?://)?([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("mysql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case mysqlTableRegex(dbName, table) => - val db = MySqlClients.get(dbName, location) - new MySqlTable(db, db.database.get, table) - case _ => throw new LocationException("not a mysql table location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClient.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClient.scala index c3b4e9b38..5434b2cf4 100644 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClient.scala @@ -14,7 +14,6 @@ package raw.sources.jdbc.oracle import oracle.net.ns.NetException import raw.utils.RawSettings -import raw.creds.api.OracleCredential import raw.sources.jdbc.api._ import java.io.Closeable @@ -28,36 +27,38 @@ import java.util.concurrent.{Executors, TimeUnit} import scala.util.control.NonFatal object OracleClient { - val timestampRegex: Regex = """timestamp\(\d+\)""".r - val interval1Regex: Regex = """interval year\(\d+\) to month""".r - val interval2Regex: Regex = """interval day\(\d+\) to second\(\d+\)""".r + private val TIMESTAMP_REGEX: Regex = """timestamp\(\d+\)""".r + private val INTERVAL1_REGEX: Regex = """interval year\(\d+\) to month""".r + private val INTERVAL2_REGEX: Regex = """interval day\(\d+\) to second\(\d+\)""".r } -class OracleClient(db: OracleCredential)(implicit settings: RawSettings) extends JdbcClient { +class OracleClient(val hostname: String, val port: Int, dbName: String, username: String, password: String)( + implicit settings: RawSettings +) extends JdbcClient { import OracleClient._ Class.forName("oracle.jdbc.OracleDriver") override val vendor: String = "oracle" - override val connectionString: String = { - val port = db.port.map(p => ":" + p).getOrElse(":1521") - s"jdbc:$vendor:thin:@${db.host}$port:${db.database}" - } - override val username: Option[String] = db.username - override val password: Option[String] = db.password + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) - override val hostname: String = db.host - override def database: Option[String] = Some(db.database) + override val maybePassword: Option[String] = Some(password) + + override val connectionString: String = { + s"jdbc:$vendor:thin:@$hostname:$port:$dbName" + } override def getConnection: Connection = { // For connection pool: // wrapSQLException(datasource.getConnection()) wrapSQLException { val props = new Properties() - username.foreach(user => props.setProperty("user", user)) - password.foreach(passwd => props.setProperty("password", passwd)) + maybeUsername.foreach(user => props.setProperty("user", user)) + maybePassword.foreach(passwd => props.setProperty("password", passwd)) // This property is defined in interface oracle.jdbc.OracleConnection.CONNECTION_PROPERTY_THIN_NET_CONNECT_TIMEOUT // see https://docs.oracle.com/cd/E18283_01/appdev.112/e13995/oracle/jdbc/OracleConnection.html#CONNECTION_PROPERTY_THIN_READ_TIMEOUT @@ -76,7 +77,7 @@ class OracleClient(db: OracleCredential)(implicit settings: RawSettings) extends super.listTables(sch) } - override def tableMetadata(database: Option[String], maybeSchema: Option[String], table: String): TableMetadata = { + override def tableMetadata(maybeSchema: Option[String], table: String): TableMetadata = { val schema = maybeSchema.get val conn = getConnection try { @@ -117,8 +118,8 @@ class OracleClient(db: OracleCredential)(implicit settings: RawSettings) extends case "long" => JdbcColumnType(INTEGER, if (nullable) 1 else 0) case "binary_float" => JdbcColumnType(REAL, if (nullable) 1 else 0) case "binary_double" => JdbcColumnType(DOUBLE, if (nullable) 1 else 0) - case timestampRegex() => JdbcColumnType(TIMESTAMP, if (nullable) 1 else 0) - case interval1Regex() | interval2Regex() => NativeIntervalType(nullable) + case TIMESTAMP_REGEX() => JdbcColumnType(TIMESTAMP, if (nullable) 1 else 0) + case INTERVAL1_REGEX() | INTERVAL2_REGEX() => NativeIntervalType(nullable) case "raw" => JdbcColumnType(BLOB, if (nullable) 1 else 0) case "blob" => JdbcColumnType(BLOB, if (nullable) 1 else 0) case _ => UnsupportedColumnType @@ -144,7 +145,7 @@ class OracleClient(db: OracleCredential)(implicit settings: RawSettings) extends case _: SocketTimeoutException => throw new RDBMSConnectTimeoutException(hostname, ex) case _: ConnectException => throw new RDBMSConnectErrorException(hostname, ex) } - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // TODO (ctm): Find documentation of Oracle error codes and check if it is best to map ORA- here. if (ex.getErrorCode == 1017) { @@ -166,9 +167,8 @@ class OracleClient(db: OracleCredential)(implicit settings: RawSettings) extends } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClients.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClients.scala deleted file mode 100644 index 0146033e7..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleClients.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import raw.client.api.LocationDescription -import raw.creds.api.OracleCredential -import raw.sources.api.{LocationException, SourceContext} - -object OracleClients { - - def get(dbName: String, location: LocationDescription)(implicit sourceContext: SourceContext): OracleClient = { - val cred: OracleCredential = location.getStringSetting("db-host") match { - case Some(host) => - val port = location.getIntSetting("db-port") - val userName = location.getStringSetting("db-username") - val password = location.getStringSetting("db-password") - OracleCredential(host, port, dbName, userName, password) - case _ => sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: OracleCredential) => cred - case _ => throw new LocationException(s"no credential found for oracle: $dbName") - } - } - new OracleClient(cred)(sourceContext.settings) - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocationBuilder.scala deleted file mode 100644 index 75bbbaf50..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class OracleLocationBuilder extends JdbcLocationBuilder { - - private val oracleDatabaseRegex = """oracle:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("oracle") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case oracleDatabaseRegex(dbName) => - val db = OracleClients.get(dbName, location) - new OracleLocation(db, db.database.get) - case _ => throw new LocationException("not an oracle database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchema.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchema.scala deleted file mode 100644 index a1128e82f..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchema.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -class OracleSchema( - cli: OracleClient, - dbName: String, - schema: String -) extends JdbcSchemaLocation(cli, Some(schema)) { - - override def rawUri: String = s"oracle:$dbName/$schema" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables(schema) - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"oracle:$dbName/$schema/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocation.scala new file mode 100644 index 000000000..a09c2bef7 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocation.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.oracle + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class OracleSchemaLocation( + cli: OracleClient, + val schema: String +) extends JdbcSchemaLocation(cli, Some(schema)) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this(host: String, port: Int, dbName: String, username: String, password: String, schema: String)( + implicit settings: RawSettings + ) = { + this( + new OracleClient(host, port, dbName, username, password), + schema + ) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables(schema) + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new OracleTableLocation(cli, schema, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocationBuilder.scala deleted file mode 100644 index 67c468044..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleSchemaLocationBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class OracleSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - override def schemes: Seq[String] = Seq("oracle") - - private val schemaRegex = """oracle:(?://)?([^/]+)/([^/]+)""".r - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName, schema) => - val db = OracleClients.get(dbName, location) - new OracleSchema(db, db.database.get, schema) - case _ => throw new LocationException("not an oracle schema location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocation.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleServerLocation.scala similarity index 50% rename from sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocation.scala rename to sources/src/main/scala/raw/sources/jdbc/oracle/OracleServerLocation.scala index b68fa8e2b..ea91af591 100644 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleServerLocation.scala @@ -14,21 +14,28 @@ package raw.sources.jdbc.oracle import java.io.Closeable import raw.sources.jdbc.api._ - -class OracleLocation( - cli: OracleClient, - dbName: String -) extends JdbcLocation(cli, "oracle", dbName) { - - override def rawUri: String = s"oracle:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { +import raw.utils.RawSettings + +class OracleServerLocation( + val host: String, + val port: Int, + val dbName: String, + val username: String, + val password: String +)( + implicit settings: RawSettings +) extends JdbcServerLocation(new OracleClient(host, port, dbName, username, password)) { + + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { + new Iterator[JdbcSchemaLocation] with Closeable { + private val cli = jdbcClient.asInstanceOf[OracleClient] private val it = cli.listSchemas override def hasNext: Boolean = it.hasNext - override def next(): String = s"oracle:$dbName/${it.next()}" + override def next(): JdbcSchemaLocation = { + new OracleSchemaLocation(cli, it.next()) + } override def close(): Unit = it.close() } diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTable.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTable.scala deleted file mode 100644 index dd4e4693d..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTable.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import raw.sources.jdbc.api.JdbcTableLocation - -class OracleTable( - cli: OracleClient, - dbName: String, - schema: String, - table: String -) extends JdbcTableLocation(cli, "oracle", dbName, table.toUpperCase, Some(schema.toUpperCase)) { - - override def rawUri: String = s"oracle:$dbName/$schema/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocation.scala new file mode 100644 index 000000000..6aef95b0e --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocation.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.oracle + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class OracleTableLocation( + cli: OracleClient, + val schema: String, + val table: String +) extends JdbcTableLocation(cli, Some(schema.toUpperCase), table.toUpperCase) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schema: String, + tableName: String + )(implicit settings: RawSettings) = { + this( + new OracleClient(host, port, dbName, username, password), + schema, + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocationBuilder.scala deleted file mode 100644 index 8cbd61fff..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/oracle/OracleTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.oracle - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class OracleTableLocationBuilder extends JdbcTableLocationBuilder { - - private val oracleTableRegex = """oracle:(?://)?([^/]+)/([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("oracle") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case oracleTableRegex(dbName, schema, table) => - val db = OracleClients.get(dbName, location) - new OracleTable(db, db.database.get, schema, table) - case _ => throw new LocationException("not an oracle table location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClient.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClient.scala index 52a07da4f..1b45f6f36 100644 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClient.scala @@ -15,14 +15,15 @@ package raw.sources.jdbc.pgsql import org.postgresql.util.PSQLException import java.util.concurrent.TimeUnit -import raw.creds.api.PostgresqlCredential import raw.sources.jdbc.api._ import raw.utils.RawSettings import java.net.{SocketTimeoutException, UnknownHostException} import scala.util.control.NonFatal -class PostgresqlClient(db: PostgresqlCredential)(implicit settings: RawSettings) extends JdbcClient { +class PostgresqlClient(val hostname: String, val port: Int, dbName: String, username: String, password: String)( + implicit settings: RawSettings +) extends JdbcClient { Class.forName("org.postgresql.Driver") @@ -31,23 +32,16 @@ class PostgresqlClient(db: PostgresqlCredential)(implicit settings: RawSettings) private val readTimeout = getReadTimeout(TimeUnit.SECONDS) override val vendor: String = "postgresql" - override val connectionString: String = { - val maybePort = db.port.map(p => ":" + p).getOrElse("") - s"jdbc:$vendor://${db.host}$maybePort/${db.database}?connectTimeout=$connectTimeout&socketTimeout=$readTimeout" - } - override val username: Option[String] = db.username - override val password: Option[String] = db.password - override val hostname: String = db.host + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) - override val database: Option[String] = Some(db.database) - // override val datasource: DataSource = { - // val pgDatasource = new PGSimpleDataSource() - // pgDatasource.setURL(connectionString) - // pgDatasource.setUser(username.orNull) - // pgDatasource.setPassword(password.orNull) - // pgDatasource - // } + override val maybePassword: Option[String] = Some(password) + + override val connectionString: String = { + s"jdbc:$vendor://$hostname:$port/$dbName?connectTimeout=$connectTimeout&socketTimeout=$readTimeout" + } override def wrapSQLException[T](f: => T): T = { try { @@ -56,7 +50,7 @@ class PostgresqlClient(db: PostgresqlCredential)(implicit settings: RawSettings) case ex: PSQLException => ex.getCause match { case _: UnknownHostException => throw new RDBMSUnknownHostException(hostname, ex) case _: SocketTimeoutException => throw new RDBMSConnectTimeoutException(hostname, ex) - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // Some more codes here (DB2 Universal Messages manual), various databases have varying degrees of compliance //https://www.ibm.com/support/knowledgecenter/en/SS6NHC/com.ibm.swg.im.dashdb.messages.doc/doc/rdb2stt.html @@ -76,9 +70,8 @@ class PostgresqlClient(db: PostgresqlCredential)(implicit settings: RawSettings) } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClients.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClients.scala deleted file mode 100644 index 9fe757d09..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlClients.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import raw.creds.api.PostgresqlCredential -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -object PostgresqlClients { - - def get(dbName: String, location: LocationDescription)(implicit sourceContext: SourceContext): PostgresqlClient = { - val cred: PostgresqlCredential = location.getStringSetting("db-host") match { - case Some(host) => - val port = location.getIntSetting("db-port") - val userName = location.getStringSetting("db-username") - val password = location.getStringSetting("db-password") - PostgresqlCredential(host, port, dbName, userName, password) - case _ => sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: PostgresqlCredential) => cred - case _ => throw new LocationException(s"no credential found for postgresql: $dbName") - } - } - new PostgresqlClient(cred)(sourceContext.settings) - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocation.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocation.scala deleted file mode 100644 index ced7b59df..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocation.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import java.io.Closeable -import raw.sources.jdbc.api._ - -class PostgresqlLocation( - cli: PostgresqlClient, - dbName: String -) extends JdbcLocation(cli, "pgsql", dbName) { - - override def rawUri: String = s"pgsql:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listSchemas - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"pgsql:$dbName/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocationBuilder.scala deleted file mode 100644 index fba19c215..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class PostgresqlLocationBuilder extends JdbcLocationBuilder { - - private val postgresqlDatabaseRegex = """pgsql:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("pgsql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case postgresqlDatabaseRegex(dbName) => - val db = PostgresqlClients.get(dbName, location) - new PostgresqlLocation(db, db.database.get) - case _ => throw new LocationException("not a postgresql database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchema.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchema.scala deleted file mode 100644 index edecc6781..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchema.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -class PostgresqlSchema( - cli: PostgresqlClient, - dbName: String, - schema: String -) extends JdbcSchemaLocation(cli, Some(schema)) { - - override def rawUri: String = s"pgsql:$dbName/$schema" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables(schema) - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"pgsql:$dbName/$schema/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocation.scala new file mode 100644 index 000000000..810d13cfa --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocation.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.pgsql + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class PostgresqlSchemaLocation( + cli: PostgresqlClient, + val schema: String +) extends JdbcSchemaLocation(cli, Some(schema)) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this(host: String, port: Int, dbName: String, username: String, password: String, schema: String)( + implicit settings: RawSettings + ) = { + this( + new PostgresqlClient(host, port, dbName, username, password), + schema + ) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables(schema) + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new PostgresqlTableLocation(cli, schema, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocationBuilder.scala deleted file mode 100644 index 5fafe2d2b..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlSchemaLocationBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class PostgresqlSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - - private val schemaRegex = """pgsql:(?://)?([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("pgsql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName, schema) => - val db = PostgresqlClients.get(dbName, location) - new PostgresqlSchema(db, db.database.get, schema) - case _ => throw new LocationException("not a postgresql schema location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlServerLocation.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlServerLocation.scala new file mode 100644 index 000000000..1273e3d46 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlServerLocation.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.pgsql + +import java.io.Closeable +import raw.sources.jdbc.api._ +import raw.utils.RawSettings + +class PostgresqlServerLocation( + val host: String, + val port: Int, + val dbName: String, + val username: String, + val password: String +)( + implicit settings: RawSettings +) extends JdbcServerLocation(new PostgresqlClient(host, port, dbName, username, password)) { + + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { + new Iterator[JdbcSchemaLocation] with Closeable { + private val cli = jdbcClient.asInstanceOf[PostgresqlClient] + private val it = cli.listSchemas + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcSchemaLocation = { + new PostgresqlSchemaLocation(cli, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTable.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTable.scala deleted file mode 100644 index a285de851..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTable.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import raw.sources.jdbc.api.JdbcTableLocation - -class PostgresqlTable( - cli: PostgresqlClient, - dbName: String, - schema: String, - table: String -) extends JdbcTableLocation(cli, "pgsql", dbName, table, Some(schema)) { - - override def rawUri: String = s"pgsql:$dbName/$schema/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocation.scala new file mode 100644 index 000000000..9915ba1c1 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocation.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.pgsql + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class PostgresqlTableLocation( + cli: PostgresqlClient, + val schema: String, + val table: String +) extends JdbcTableLocation(cli, Some(schema), table) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schema: String, + tableName: String + )(implicit settings: RawSettings) = { + this( + new PostgresqlClient(host, port, dbName, username, password), + schema, + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocationBuilder.scala deleted file mode 100644 index 77cd3c6ea..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/pgsql/PostgresqlTableLocationBuilder.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.pgsql - -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} -import raw.sources.api.{LocationException, SourceContext} - -class PostgresqlTableLocationBuilder extends JdbcTableLocationBuilder { - - private val postgresqlTableRegex = """pgsql:(?://)?([^/]+)/([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("pgsql") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case postgresqlTableRegex(dbName, schema, table) => - val db = PostgresqlClients.get(dbName, location) - new PostgresqlTable(db, db.database.get, schema, table) - case _ => throw new LocationException("not a postgresql location") - } - - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClient.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClient.scala index 574f74679..a476898a9 100644 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClient.scala @@ -14,7 +14,6 @@ package raw.sources.jdbc.snowflake import net.snowflake.client.jdbc.SnowflakeSQLException import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState -import raw.creds.api.SnowflakeCredential import raw.sources.jdbc.api._ import raw.utils.RawSettings @@ -23,28 +22,39 @@ import java.util.concurrent.{Executors, TimeUnit} import java.util.{Properties, TimeZone} import scala.util.control.NonFatal -class SnowflakeClient(db: SnowflakeCredential)(implicit settings: RawSettings) extends JdbcClient { +class SnowflakeClient( + dbName: String, + username: String, + password: String, + val accountIdentifier: String, + val parameters: Map[String, String] +)( + implicit settings: RawSettings +) extends JdbcClient { Class.forName("net.snowflake.client.jdbc.SnowflakeDriver") + override val hostname: String = s"$accountIdentifier.snowflakecomputing.com" + override val vendor: String = "snowflake" - override val connectionString: String = s"jdbc:snowflake://${db.host}/" + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) + + override val maybePassword: Option[String] = Some(password) - override val username: Option[String] = db.username - override val password: Option[String] = db.password + override val connectionString: String = s"jdbc:snowflake://$hostname/" - override val hostname: String = db.host - override val database: Option[String] = Some(db.database) override def getConnection: Connection = { wrapSQLException { - val parameters = db.parameters ++ Seq("db" -> db.database) + val params = parameters ++ Seq("db" -> dbName) val props = new Properties() - username.foreach(user => props.setProperty("user", user)) - password.foreach(passwd => props.setProperty("password", passwd)) + maybeUsername.foreach(user => props.setProperty("user", user)) + maybePassword.foreach(passwd => props.setProperty("password", passwd)) props.setProperty("JDBC_QUERY_RESULT_FORMAT", "JSON") - for ((key, value) <- parameters) props.setProperty(key, value) + for ((key, value) <- params) props.setProperty(key, value) // (CTM) I am having issues with sql.Time with timezones. I am seeing a shift if the timezone was not set to UTC. TimeZone.setDefault(TimeZone.getTimeZone("UTC")) @@ -55,8 +65,18 @@ class SnowflakeClient(db: SnowflakeCredential)(implicit settings: RawSettings) e } } - override def tableMetadata(database: Option[String], maybeSchema: Option[String], table: String): TableMetadata = { - super.tableMetadata(None, maybeSchema, table) + override def tableMetadata(maybeSchema: Option[String], table: String): TableMetadata = { + val conn = getConnection + try { + val res = getTableMetadata(conn, None, maybeSchema, table) + try { + getTableTypeFromTableMetadata(res) + } finally { + wrapSQLException(res.close()) + } + } finally { + wrapSQLException(conn.close()) + } } override def wrapSQLException[T](f: => T): T = { @@ -70,11 +90,11 @@ class SnowflakeClient(db: SnowflakeCredential)(implicit settings: RawSettings) e case SqlState.INVALID_PASSWORD | SqlState.INVALID_AUTHORIZATION_SPECIFICATION => throw new AuthenticationFailedException(ex) case SqlState.IO_ERROR => throw new JdbcLocationException( - s"IO error connecting to ${db.accountIdentifier}: ${ex.getMessage}", + s"IO error connecting to $accountIdentifier: ${ex.getMessage}", ex ) case SqlState.SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION => throw new JdbcLocationException( - s"unable to establish connection to ${db.accountIdentifier}: ${ex.getMessage}", + s"unable to establish connection to $accountIdentifier: ${ex.getMessage}", ex ) case SqlState.CONNECTION_EXCEPTION | SqlState.CONNECTION_FAILURE => @@ -86,10 +106,8 @@ class SnowflakeClient(db: SnowflakeCredential)(implicit settings: RawSettings) e logger.warn(s"Unexpected SQL error (code: ${ex.getErrorCode}; state: ${ex.getSQLState}).", ex) throw new JdbcLocationException(ex.getMessage, ex) } - - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } } diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClients.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClients.scala deleted file mode 100644 index f47e45d0f..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeClients.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import raw.creds.api.SnowflakeCredential -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -object SnowflakeClients { - - def get(dbName: String, location: LocationDescription)(implicit sourceContext: SourceContext): SnowflakeClient = { - - val cred = location.getStringSetting("db-account-id") match { - case Some(account) => - val userName = location.getStringSetting("db-username") - val password = location.getStringSetting("db-password") - val settings = location.getKVSetting("db-options").getOrElse(Array.empty) - SnowflakeCredential(account, dbName, userName, password, settings.toMap) - case _ => sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: SnowflakeCredential) => cred - case _ => throw new LocationException(s"no credential found for Snowflake: $dbName") - } - } - - new SnowflakeClient(cred)(sourceContext.settings) - - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocation.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocation.scala deleted file mode 100644 index df79a7c0c..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocation.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import java.io.Closeable -import raw.sources.jdbc.api._ - -class SnowflakeLocation( - cli: SnowflakeClient, - dbName: String -) extends JdbcLocation(cli, "snowflake", dbName) { - - override def rawUri: String = s"snowflake:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listSchemas - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"snowflake:$dbName/${it.next()}" - - override def close(): Unit = it.close() - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocationBuilder.scala deleted file mode 100644 index 69f90d2dc..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class SnowflakeLocationBuilder extends JdbcLocationBuilder { - - private val snowflakeTableRegex = """snowflake:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("snowflake") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case snowflakeTableRegex(dbName) => - val db = SnowflakeClients.get(dbName, location) - new SnowflakeLocation(db, db.database.get) - case _ => throw new LocationException("not an snowflake database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchema.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchema.scala deleted file mode 100644 index 80fd7dfab..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchema.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -class SnowflakeSchema( - cli: SnowflakeClient, - dbName: String, - schema: String -) extends JdbcSchemaLocation(cli, Some(schema)) { - - override def rawUri: String = s"snowflake:$dbName/$schema" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables(schema) - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"snowflake:$dbName/$schema/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocation.scala new file mode 100644 index 000000000..09c9d69c6 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocation.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.snowflake + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class SnowflakeSchemaLocation( + cli: SnowflakeClient, + val schema: String +) extends JdbcSchemaLocation(cli, Some(schema)) { + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + val accountIdentifier: String = cli.accountIdentifier + + val parameters: Map[String, String] = cli.parameters + + def this( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String], + schema: String + )(implicit settings: RawSettings) = { + this( + new SnowflakeClient( + dbName, + username, + password, + accountIdentifier, + parameters + ), + schema + ) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables(schema) + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new SnowflakeTableLocation(cli, schema, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocationBuilder.scala deleted file mode 100644 index e029627e6..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeSchemaLocationBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class SnowflakeSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - override def schemes: Seq[String] = Seq("snowflake") - - private val schemaRegex = """snowflake:(?://)?([^/]+)/([^/]+)""".r - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName, schema) => - val db = SnowflakeClients.get(dbName, location) - new SnowflakeSchema(db, db.database.get, schema) - case _ => throw new LocationException("not an snowflake schema location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeServerLocation.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeServerLocation.scala new file mode 100644 index 000000000..8b08bb87c --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeServerLocation.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.snowflake + +import java.io.Closeable +import raw.sources.jdbc.api._ +import raw.utils.RawSettings +import scala.collection.JavaConverters._ + +class SnowflakeServerLocation( + val dbName: String, + val username: String, + val password: String, + val accountIdentifier: String, + val parameters: Map[String, String] +)(implicit settings: RawSettings) + extends JdbcServerLocation( + new SnowflakeClient( + dbName, + username, + password, + accountIdentifier, + parameters + ) + ) { + + // Constructor for Java. + def this( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: java.util.Map[String, String], + settings: RawSettings + ) = this(dbName, username, password, accountIdentifier, parameters.asScala.toMap)(settings) + + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { + new Iterator[JdbcSchemaLocation] with Closeable { + private val cli = jdbcClient.asInstanceOf[SnowflakeClient] + private val it = cli.listSchemas + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcSchemaLocation = { + new SnowflakeSchemaLocation(cli, it.next()) + } + + override def close(): Unit = it.close() + } + } +} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTable.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTable.scala deleted file mode 100644 index c4cf3b32b..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTable.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import raw.sources.jdbc.api.JdbcTableLocation - -class SnowflakeTable( - cli: SnowflakeClient, - dbName: String, - schema: String, - table: String -) extends JdbcTableLocation(cli, "snowflake", dbName, table.toUpperCase, Some(schema.toUpperCase)) { - - override def rawUri: String = s"snowflake:$dbName/$schema/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocation.scala new file mode 100644 index 000000000..a9b3a0bf0 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocation.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.snowflake + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class SnowflakeTableLocation( + cli: SnowflakeClient, + val schema: String, + val table: String +) extends JdbcTableLocation(cli, Some(schema.toUpperCase), table.toUpperCase) { + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + val accountIdentifier: String = cli.accountIdentifier + + val parameters: Map[String, String] = cli.parameters + + def this( + dbName: String, + username: String, + password: String, + accountIdentifier: String, + parameters: Map[String, String], + schema: String, + tableName: String + )(implicit settings: RawSettings) = { + this( + new SnowflakeClient( + dbName, + username, + password, + accountIdentifier, + parameters + ), + schema, + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocationBuilder.scala deleted file mode 100644 index 85a530775..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/snowflake/SnowflakeTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.snowflake - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class SnowflakeTableLocationBuilder extends JdbcTableLocationBuilder { - - private val snowflakeTableRegex = """snowflake:(?://)?([^/]+)/([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("snowflake") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case snowflakeTableRegex(dbName, schema, table) => - val db = SnowflakeClients.get(dbName, location) - new SnowflakeTable(db, db.database.get, schema, table) - case _ => throw new LocationException("not an snowflake table location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClient.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClient.scala index db2eadc73..c515f7d8a 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClient.scala @@ -12,38 +12,47 @@ package raw.sources.jdbc.sqlite +import raw.sources.api.LocationException import raw.sources.jdbc.api._ import raw.utils.RawSettings -import java.nio.file.{InvalidPathException, Path} +import java.nio.file.{InvalidPathException, Path, Paths} import java.sql.SQLException import scala.util.control.NonFatal -class SqliteClient(path: Path)(implicit settings: RawSettings) extends JdbcClient { +class SqliteClient(val path: String)(implicit settings: RawSettings) extends JdbcClient { - Class.forName("org.sqlite.JDBC") + private val localPath = + try { + Paths.get(path) + } catch { + case _: InvalidPathException => throw new LocationException("invalid path") + } - /** - * The JDBC driver requires a local path. Opted to validate here instead of having constructor take a Path and - * force all the callers to valid the path, so that we have more coherent error handling. This way, if there is - * any error with the "connection string" (the path), we throw the proper exception. - * It is possible the "path" is a RAW URI to the local file, when the reader/writer exp is coming from - * readOrCacheCreate - */ + // The JDBC driver requires a local path. + // Opted to validate here instead of having constructor take a Path and force all the callers to valid the path, + // so that we have more coherent error handling. + // This way, if there is any error with the "connection string" (the path in this case), we throw early an exception. val sqlitePath: Path = try { - path.toAbsolutePath + localPath.toAbsolutePath } catch { - case ex: InvalidPathException => throw new JdbcLocationException(s"invalid local path: $path", ex) + case ex: InvalidPathException => throw new JdbcLocationException(s"invalid local path: $localPath", ex) } + Class.forName("org.sqlite.JDBC") + override val vendor: String = "sqlite" + + override val maybeUsername: Option[String] = None + + override val maybePassword: Option[String] = None + + override val maybeDatabase: Option[String] = None + override val connectionString: String = s"jdbc:$vendor:$sqlitePath" - override val username: Option[String] = None - override val password: Option[String] = None - override val database: Option[String] = None - override val hostname: String = path.toAbsolutePath.toString + override val hostname: String = localPath.toAbsolutePath.toString override def wrapSQLException[T](f: => T): T = { try { @@ -51,7 +60,7 @@ class SqliteClient(path: Path)(implicit settings: RawSettings) extends JdbcClien } catch { // TODO (ctm): check Sqlite exceptions case ex: SQLException => ex.getCause match { - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // Some more codes here (DB2 Universal Messages manual), various databases have varying degrees of compliance //https://www.ibm.com/support/knowledgecenter/en/SS6NHC/com.ibm.swg.im.dashdb.messages.doc/doc/rdb2stt.html @@ -69,9 +78,8 @@ class SqliteClient(path: Path)(implicit settings: RawSettings) extends JdbcClien } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClients.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClients.scala deleted file mode 100644 index 53ce7af71..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteClients.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlite - -import raw.client.api.LocationDescription -import raw.sources.bytestream.api.ByteStreamLocationProvider -import raw.sources.api.SourceContext - -object SqliteClients { - - def get(url: String)(implicit sourceContext: SourceContext): SqliteClient = { - // TODO (msb): This construction is suspicious... - val locationDescription = LocationDescription(url) - val location = ByteStreamLocationProvider.build(locationDescription) - val localPath = location.getLocalPath() - new SqliteClient(localPath)(sourceContext.settings) - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocationBuilder.scala deleted file mode 100644 index 77c6d81b6..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlite - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class SqliteLocationBuilder extends JdbcLocationBuilder { - - private val sqliteRegex = """sqlite:(?://)?(.+)""".r - - override def schemes: Seq[String] = Seq("sqlite") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case sqliteRegex(locationUrl) => - val db = SqliteClients.get(locationUrl) - new SqliteLocation(db, locationUrl) - case _ => throw new LocationException("not a sqlite database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchema.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocation.scala similarity index 57% rename from sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchema.scala rename to sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocation.scala index 1e98dbc97..8232a6425 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchema.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocation.scala @@ -13,23 +13,28 @@ package raw.sources.jdbc.sqlite import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings -class SqliteSchema( +class SqliteSchemaLocation( cli: SqliteClient ) extends JdbcSchemaLocation(cli, None) { - private val path = cli.sqlitePath.toString + val path: String = cli.path - override def rawUri: String = s"sqlite:$path" + def this(path: String)(implicit settings: RawSettings) = { + this(new SqliteClient(path)) + } - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { private val it = cli.listTables("") override def hasNext: Boolean = it.hasNext - override def next(): String = s"sqlite:${it.next()}:file:$path" + override def next(): JdbcTableLocation = { + new SqliteTableLocation(cli, it.next()) + } override def close(): Unit = it.close() } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocationBuilder.scala deleted file mode 100644 index 12cbfef2c..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteSchemaLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlite - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class SqliteSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - - private val sqliteRegex = """sqlite:(?://)?(.+)""".r - - override def schemes: Seq[String] = Seq("sqlite") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case sqliteRegex(locationUrl) => - val db = SqliteClients.get(locationUrl) - new SqliteSchema(db) - case _ => throw new LocationException("not a sqlite schema location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocation.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteServerLocation.scala similarity index 66% rename from sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocation.scala rename to sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteServerLocation.scala index 30e48d4a7..d4d508d0e 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteServerLocation.scala @@ -14,17 +14,13 @@ package raw.sources.jdbc.sqlite import java.io.Closeable import raw.sources.jdbc.api._ +import raw.utils.RawSettings -class SqliteLocation( - cli: SqliteClient, - dbName: String -) extends JdbcLocation(cli, "sqlite", dbName) { +class SqliteServerLocation(val path: String)( + implicit settings: RawSettings +) extends JdbcServerLocation(new SqliteClient(path)) { - private val path = cli.sqlitePath.toString - - override def rawUri: String = s"sqlite:$path" - - override def listSchemas(): Iterator[String] with Closeable = { + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { throw new JdbcLocationException("no schemas in sqlite") } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTable.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocation.scala similarity index 62% rename from sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTable.scala rename to sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocation.scala index 5d51f6422..d33a263b4 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTable.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocation.scala @@ -13,15 +13,17 @@ package raw.sources.jdbc.sqlite import raw.sources.jdbc.api._ +import raw.utils.RawSettings -class SqliteTable( +class SqliteTableLocation( cli: SqliteClient, - dbName: String, - table: String -) extends JdbcTableLocation(cli, "sqlite", dbName, table, None) { + val table: String +) extends JdbcTableLocation(cli, None, table) { - private val path = cli.sqlitePath.toString + val path: String = cli.path - override def rawUri: String = s"sqlite:$table:$path" + def this(path: String, table: String)(implicit settings: RawSettings) = { + this(new SqliteClient(path), table) + } } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocationBuilder.scala deleted file mode 100644 index 071257685..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlite/SqliteTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlite - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class SqliteTableLocationBuilder extends JdbcTableLocationBuilder { - - private val sqliteRegex = """sqlite:(?://)?([^:]+):(.+)""".r - - override def schemes: Seq[String] = Seq("sqlite") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case sqliteRegex(table, locationUrl) => - val db = SqliteClients.get(locationUrl) - new SqliteTable(db, locationUrl, table) - case _ => throw new LocationException("not a sqlite table location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClient.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClient.scala index 9ebe70fe3..598c8b5f4 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClient.scala @@ -15,14 +15,15 @@ package raw.sources.jdbc.sqlserver import com.microsoft.sqlserver.jdbc.SQLServerException import java.util.concurrent.TimeUnit -import raw.creds.api.SqlServerCredential import raw.sources.jdbc.api._ import raw.utils.RawSettings import java.net.{SocketTimeoutException, UnknownHostException} import scala.util.control.NonFatal -class SqlServerClient(protected val db: SqlServerCredential)(implicit settings: RawSettings) extends JdbcClient { +class SqlServerClient(val hostname: String, val port: Int, dbName: String, username: String, password: String)( + implicit settings: RawSettings +) extends JdbcClient { Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver") @@ -33,16 +34,18 @@ class SqlServerClient(protected val db: SqlServerCredential)(implicit settings: private val readTimeout = getReadTimeout(TimeUnit.MILLISECONDS) override val vendor: String = "sqlserver" + + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) + + override val maybePassword: Option[String] = Some(password) + override val connectionString: String = { - val maybePort = db.port.map(p => ":" + p).getOrElse("") - // explicit sendTimeAsDatetime=false to support time JDBC parameters (https://github.com/microsoft/mssql-jdbc/issues/559) - s"jdbc:$vendor://${db.host}$maybePort;databaseName=${db.database};loginTimeout=$connectTimeout;socketTimeout=$readTimeout;sendTimeAsDatetime=false" + // Explicit sendTimeAsDatetime=false to support time JDBC parameters (https://github.com/microsoft/mssql-jdbc/issues/559) + s"jdbc:$vendor://$hostname:$port;databaseName=$dbName;loginTimeout=$connectTimeout;socketTimeout=$readTimeout;sendTimeAsDatetime=false" } - override val username: Option[String] = db.username - override val password: Option[String] = db.password - override val hostname: String = db.host - override val database: Option[String] = Some(db.database) override def wrapSQLException[T](f: => T): T = { try { f @@ -51,7 +54,7 @@ class SqlServerClient(protected val db: SqlServerCredential)(implicit settings: case ex: SQLServerException => ex.getCause match { case _: UnknownHostException => throw new RDBMSUnknownHostException(hostname, ex) case _: SocketTimeoutException => throw new RDBMSConnectTimeoutException(hostname, ex) - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // Some more codes here (DB2 Universal Messages manual), various databases have varying degrees of compliance //https://www.ibm.com/support/knowledgecenter/en/SS6NHC/com.ibm.swg.im.dashdb.messages.doc/doc/rdb2stt.html @@ -71,9 +74,8 @@ class SqlServerClient(protected val db: SqlServerCredential)(implicit settings: } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClients.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClients.scala deleted file mode 100644 index 0150a3b7b..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerClients.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import raw.creds.api.SqlServerCredential -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription - -object SqlServerClients { - - def get(dbName: String, location: LocationDescription)(implicit sourceContext: SourceContext): SqlServerClient = { - val cred: SqlServerCredential = location.getStringSetting("db-host") match { - case Some(host) => - val port = location.getIntSetting("db-port") - val userName = location.getStringSetting("db-username") - val password = location.getStringSetting("db-password") - SqlServerCredential(host, port, dbName, userName, password) - case _ => sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: SqlServerCredential) => cred - case _ => throw new LocationException(s"no credential found for sqlserver: $dbName") - } - } - new SqlServerClient(cred)(sourceContext.settings) - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocationBuilder.scala deleted file mode 100644 index 15a8cdc6c..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class SqlServerLocationBuilder extends JdbcLocationBuilder { - - private val sqlServerTableRegex = """sqlserver:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("sqlserver") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case sqlServerTableRegex(dbName) => - val db = SqlServerClients.get(dbName, location) - new SqlServerLocation(db, db.database.get) - case _ => throw new LocationException("not a sqlserver database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchema.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchema.scala deleted file mode 100644 index 1ae5c8473..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchema.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -class SqlServerSchema( - cli: SqlServerClient, - dbName: String, - schema: String -) extends JdbcSchemaLocation(cli, Some(schema)) { - - override def rawUri: String = s"sqlserver:$dbName/$schema" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables(schema) - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"sqlserver:$dbName/$schema/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocation.scala new file mode 100644 index 000000000..6490022c5 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocation.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.sqlserver + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class SqlServerSchemaLocation( + cli: SqlServerClient, + val schema: String +) extends JdbcSchemaLocation(cli, Some(schema)) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this(host: String, port: Int, dbName: String, username: String, password: String, schema: String)( + implicit settings: RawSettings + ) = { + this( + new SqlServerClient(host, port, dbName, username, password), + schema + ) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables(schema) + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new SqlServerTableLocation(cli, schema, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocationBuilder.scala deleted file mode 100644 index e183a90e7..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerSchemaLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class SqlServerSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - - private val schemaRegex = """sqlserver:(?://)?([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("sqlserver") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName, schema) => - val db = SqlServerClients.get(dbName, location) - new SqlServerSchema(db, db.database.get, schema) - case _ => throw new LocationException("not a sqlserver schema location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocation.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerServerLocation.scala similarity index 50% rename from sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocation.scala rename to sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerServerLocation.scala index 0b2b0c37c..192c51866 100644 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerLocation.scala +++ b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerServerLocation.scala @@ -14,21 +14,28 @@ package raw.sources.jdbc.sqlserver import java.io.Closeable import raw.sources.jdbc.api._ - -class SqlServerLocation( - cli: SqlServerClient, - dbName: String -) extends JdbcLocation(cli, "sqlserver", dbName) { - - override def rawUri: String = s"sqlserver:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { +import raw.utils.RawSettings + +class SqlServerServerLocation( + val host: String, + val port: Int, + val dbName: String, + val username: String, + val password: String +)( + implicit settings: RawSettings +) extends JdbcServerLocation(new SqlServerClient(host, port, dbName, username, password)) { + + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { + new Iterator[JdbcSchemaLocation] with Closeable { + private val cli = jdbcClient.asInstanceOf[SqlServerClient] private val it = cli.listSchemas override def hasNext: Boolean = it.hasNext - override def next(): String = s"sqlserver:$dbName/${it.next()}" + override def next(): JdbcSchemaLocation = { + new SqlServerSchemaLocation(cli, it.next()) + } override def close(): Unit = it.close() } diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTable.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTable.scala deleted file mode 100644 index a2a1ac16a..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTable.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import raw.sources.jdbc.api.JdbcTableLocation - -class SqlServerTable( - cli: SqlServerClient, - dbName: String, - schema: String, - table: String -) extends JdbcTableLocation(cli, "sqlserver", dbName, table, Some(schema)) { - - override def rawUri: String = s"sqlserver:$dbName/$schema/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocation.scala new file mode 100644 index 000000000..a130fe51a --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocation.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.sqlserver + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class SqlServerTableLocation( + cli: SqlServerClient, + val schema: String, + val table: String +) extends JdbcTableLocation(cli, Some(schema), table) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + def this( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schema: String, + tableName: String + )(implicit settings: RawSettings) = { + this( + new SqlServerClient(host, port, dbName, username, password), + schema, + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocationBuilder.scala deleted file mode 100644 index e927ae9a4..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/sqlserver/SqlServerTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.sqlserver - -import raw.sources.api.{LocationException, SourceContext} -import raw.client.api.LocationDescription -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class SqlServerTableLocationBuilder extends JdbcTableLocationBuilder { - - private val sqlServerTableRegex = """sqlserver:(?://)?([^/]+)/([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("sqlserver") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case sqlServerTableRegex(dbName, schema, table) => - val db = SqlServerClients.get(dbName, location) - new SqlServerTable(db, db.database.get, schema, table) - case _ => throw new LocationException("not a sqlserver table location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClient.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClient.scala index 65faf0962..80eca236d 100644 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClient.scala +++ b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClient.scala @@ -13,7 +13,6 @@ package raw.sources.jdbc.teradata import raw.utils.RawSettings -import raw.creds.api.TeradataCredential import raw.sources.jdbc.api._ import java.net.{NoRouteToHostException, SocketTimeoutException, UnknownHostException} @@ -21,35 +20,40 @@ import java.sql.{Connection, DriverManager, ResultSetMetaData, SQLException} import scala.collection.mutable import scala.util.control.NonFatal -class TeradataClient(db: TeradataCredential)(implicit settings: RawSettings) extends JdbcClient { +class TeradataClient( + val hostname: String, + val port: Int, + dbName: String, + username: String, + password: String, + val parameters: Map[String, String] +)( + implicit settings: RawSettings +) extends JdbcClient { Class.forName("com.teradata.jdbc.TeraDriver") override val vendor: String = "teradata" + + override val maybeDatabase: Option[String] = Some(dbName) + + override val maybeUsername: Option[String] = Some(username) + + override val maybePassword: Option[String] = Some(password) + override val connectionString: String = { - // (ctm) Received null parameters while running rawcli tests. - val params: Seq[(String, String)] = Option(db.parameters).getOrElse(Map.empty).toSeq ++ - db.port.map(port => Seq(("DBS_PORT", port.toString))).getOrElse(Seq.empty) - if (params.nonEmpty) { - s"jdbc:$vendor://${db.host}/${params.map(p => s"${p._1}=${p._2}").mkString(",")}" - } else { - s"jdbc:$vendor://${db.host}" - } + val params = parameters + ("DBS_PORT" -> port.toString) + s"jdbc:$vendor://$hostname/${params.map(p => s"${p._1}=${p._2}").mkString(",")}" } - override val username: Option[String] = db.username - override val password: Option[String] = db.password - - override val hostname: String = db.host - override val database: Option[String] = None override def getConnection: Connection = { wrapSQLException { // Teradata jdbc connections does not have the setNetworkTimeout - DriverManager.getConnection(connectionString, username.orNull, password.orNull) + DriverManager.getConnection(connectionString, maybeUsername.orNull, maybePassword.orNull) } } - override def tableMetadata(database: Option[String], maybeSchema: Option[String], table: String): TableMetadata = { + override def tableMetadata(maybeSchema: Option[String], table: String): TableMetadata = { val schema = maybeSchema.get val conn = getConnection try { @@ -85,7 +89,7 @@ class TeradataClient(db: TeradataCredential)(implicit settings: RawSettings) ext // RuntimeErrorsSourceTeradataTestCase.test_register_bad_port_timeout the host is correct but the port is wrong. throw new RDBMSUnknownHostException(hostname, ex) case _: SocketTimeoutException => throw new RDBMSConnectTimeoutException(hostname, ex) - case int: InterruptedException => throw int + case ex: InterruptedException => throw ex case _ => // Some more codes here (DB2 Universal Messages manual), various databases have varying degrees of compliance //https://www.ibm.com/support/knowledgecenter/en/SS6NHC/com.ibm.swg.im.dashdb.messages.doc/doc/rdb2stt.html @@ -105,9 +109,8 @@ class TeradataClient(db: TeradataCredential)(implicit settings: RawSettings) ext } } case ex: JdbcLocationException => throw ex - case NonFatal(t) => - logger.warn("Unexpected SQL error.", t) - throw new JdbcLocationException(s"unexpected database error", t) + case ex: InterruptedException => throw ex + case NonFatal(t) => throw new JdbcLocationException(s"unexpected database error", t) } } diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClients.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClients.scala deleted file mode 100644 index 9509257e7..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataClients.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.creds.api.TeradataCredential -import raw.sources.api.{LocationException, SourceContext} - -object TeradataClients { - - def get(dbName: String)(implicit sourceContext: SourceContext): TeradataClient = { - sourceContext.credentialsService.getRDBMSServer(sourceContext.user, dbName) match { - case Some(cred: TeradataCredential) => new TeradataClient(cred)(sourceContext.settings) - case _ => throw new LocationException(s"no credential found for teradata: $dbName") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocation.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocation.scala deleted file mode 100644 index 7251028b5..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocation.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.sources.jdbc.api.JdbcLocation - -import java.io.Closeable - -class TeradataLocation( - cli: TeradataClient, - dbName: String -) extends JdbcLocation(cli, "teradata", dbName) { - - override def rawUri: String = s"teradata:$dbName" - - override def listSchemas(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listSchemas - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"teradata:$dbName/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocationBuilder.scala deleted file mode 100644 index b423d753c..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcLocation, JdbcLocationBuilder} - -class TeradataLocationBuilder extends JdbcLocationBuilder { - - private val teradataDbRegex = """teradata:(?://)?([^/]+)""".r - - override def schemes: Seq[String] = Seq("teradata") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcLocation = { - location.url match { - case teradataDbRegex(dbName) => - val db = TeradataClients.get(dbName) - new TeradataLocation(db, dbName) - case _ => throw new LocationException("not a teradata database location") - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchema.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchema.scala deleted file mode 100644 index 5d86483a8..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchema.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import java.io.Closeable -import raw.sources.jdbc.api.JdbcSchemaLocation - -// This might be misleading, this is a Teradata database but works in a similar way to a Oracle schema -// so just remember that like oracle users are also 'databases/schemas' -class TeradataSchema( - cli: TeradataClient, - dbName: String, - schema: String -) extends JdbcSchemaLocation(cli, Some(schema)) { - - override def rawUri: String = s"teradata:$dbName/$schema" - - override def listTables(): Iterator[String] with Closeable = { - new Iterator[String] with Closeable { - private val it = cli.listTables(schema) - - override def hasNext: Boolean = it.hasNext - - override def next(): String = s"teradata:$dbName/$schema/${it.next()}" - - override def close(): Unit = it.close() - } - } - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocation.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocation.scala new file mode 100644 index 000000000..d06133893 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocation.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.teradata + +import java.io.Closeable +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcTableLocation} +import raw.utils.RawSettings + +class TeradataSchemaLocation( + cli: TeradataClient, + val schema: String +) extends JdbcSchemaLocation(cli, Some(schema)) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + val parameters: Map[String, String] = cli.parameters + + def this( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schema: String, + parameters: Map[String, String] + )(implicit settings: RawSettings) = { + this( + new TeradataClient(host, port, dbName, username, password, parameters), + schema + ) + } + + override def listTables(): Iterator[JdbcTableLocation] with Closeable = { + new Iterator[JdbcTableLocation] with Closeable { + private val it = cli.listTables(schema) + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcTableLocation = { + new TeradataTableLocation(cli, schema, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocationBuilder.scala deleted file mode 100644 index 7ece8b0b7..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataSchemaLocationBuilder.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcSchemaLocationBuilder} - -class TeradataSchemaLocationBuilder extends JdbcSchemaLocationBuilder { - - private val schemaRegex = """teradata:(?://)?([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("teradata") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcSchemaLocation = { - location.url match { - case schemaRegex(dbName, schema) => - val db = TeradataClients.get(dbName) - new TeradataSchema(db, dbName, schema) - case _ => throw new LocationException("not a teradata schema location") - } - } -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataServerLocation.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataServerLocation.scala new file mode 100644 index 000000000..766589348 --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataServerLocation.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.teradata + +import raw.sources.jdbc.api.{JdbcSchemaLocation, JdbcServerLocation} +import raw.utils.RawSettings + +import java.io.Closeable + +class TeradataServerLocation( + val host: String, + val port: Int, + val dbName: String, + val username: String, + val password: String, + val parameters: Map[String, String] +)( + implicit settings: RawSettings +) extends JdbcServerLocation(new TeradataClient(host, port, dbName, username, password, parameters)) { + + override def listSchemas(): Iterator[JdbcSchemaLocation] with Closeable = { + new Iterator[JdbcSchemaLocation] with Closeable { + private val cli = jdbcClient.asInstanceOf[TeradataClient] + private val it = cli.listSchemas + + override def hasNext: Boolean = it.hasNext + + override def next(): JdbcSchemaLocation = { + new TeradataSchemaLocation(cli, it.next()) + } + + override def close(): Unit = it.close() + } + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTable.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTable.scala deleted file mode 100644 index b2754b4e6..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTable.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.sources.jdbc.api.JdbcTableLocation - -class TeradataTable( - cli: TeradataClient, - dbName: String, - schema: String, - table: String -) extends JdbcTableLocation(cli, "teradata", dbName, table, Some(schema)) { - - override def rawUri: String = s"teradata:$dbName/$schema/$table" - -} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocation.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocation.scala new file mode 100644 index 000000000..471b324de --- /dev/null +++ b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocation.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.sources.jdbc.teradata + +import raw.sources.jdbc.api.JdbcTableLocation +import raw.utils.RawSettings + +class TeradataTableLocation( + cli: TeradataClient, + val schema: String, + val table: String +) extends JdbcTableLocation(cli, Some(schema), table) { + + val host: String = cli.hostname + + val port: Int = cli.port + + val dbName: String = cli.maybeDatabase.get + + val username: String = cli.maybeUsername.get + + val password: String = cli.maybePassword.get + + val parameters: Map[String, String] = cli.parameters + + def this( + host: String, + port: Int, + dbName: String, + username: String, + password: String, + schema: String, + tableName: String, + parameters: Map[String, String] + )(implicit settings: RawSettings) = { + this( + new TeradataClient(host, port, dbName, username, password, parameters), + schema, + tableName + ) + } + +} diff --git a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocationBuilder.scala b/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocationBuilder.scala deleted file mode 100644 index 3deda23d9..000000000 --- a/sources/src/main/scala/raw/sources/jdbc/teradata/TeradataTableLocationBuilder.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.jdbc.teradata - -import raw.client.api.LocationDescription -import raw.sources.api.{LocationException, SourceContext} -import raw.sources.jdbc.api.{JdbcTableLocation, JdbcTableLocationBuilder} - -class TeradataTableLocationBuilder extends JdbcTableLocationBuilder { - - private val teradataTableRegex = """teradata:(?://)?([^/]+)/([^/]+)/([^/]+)""".r - - override def schemes: Seq[String] = Seq("teradata") - - override def build(location: LocationDescription)(implicit sourceContext: SourceContext): JdbcTableLocation = { - location.url match { - case teradataTableRegex(dbName, schema, table) => - val db = TeradataClients.get(dbName) - new TeradataTable(db, dbName, schema, table) - case _ => throw new LocationException("not a teradata location") - } - } - -} diff --git a/sources/src/test/scala/raw/auth/api/AuthTestUsers.scala b/sources/src/test/scala/raw/auth/api/AuthTestUsers.scala deleted file mode 100644 index 393ea7954..000000000 --- a/sources/src/test/scala/raw/auth/api/AuthTestUsers.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.auth.api - -import raw.utils._ - -trait AuthTestUsers { - this: RawTestSuite with SettingsTestContext => - - val TestJoe = InteractiveUser(Uid("joeUid"), "Joe Smith", "joe@example.com") - val TestJane = InteractiveUser(Uid("janeUid"), "Jane Smith", "jane@example.com") - val TestEscapeChars = InteractiveUser(Uid("escape_chars@|uid"), "Escape Chars", "escape-chars@example.com") - -} diff --git a/sources/src/test/scala/raw/creds/api/CredentialsTestContext.scala b/sources/src/test/scala/raw/creds/api/CredentialsTestContext.scala deleted file mode 100644 index 8422cc498..000000000 --- a/sources/src/test/scala/raw/creds/api/CredentialsTestContext.scala +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.api - -import org.scalatest.BeforeAndAfterAll -import raw.utils.{AuthenticatedUser, RawTestSuite, RawUtils, SettingsTestContext} - -import java.time.temporal.ChronoUnit -import scala.collection.mutable - -trait CredentialsTestContext extends BeforeAndAfterAll { - this: RawTestSuite with SettingsTestContext => - - private var instance: CredentialsService = _ - - private val newHttpCreds = new mutable.ArrayBuffer[(AuthenticatedUser, (String, NewHttpCredential))]() - private val s3Buckets = new mutable.HashSet[(AuthenticatedUser, S3Bucket)]() - private val rdbmsServers = new mutable.HashMap[(AuthenticatedUser, String), RelationalDatabaseCredential]() - private val dropboxTokens = new mutable.HashMap[AuthenticatedUser, DropboxToken]() - private val secrets = new mutable.HashSet[(AuthenticatedUser, Secret)]() - - def rdbms(user: AuthenticatedUser, name: String, db: RelationalDatabaseCredential): Unit = { - assert(rdbmsServers.put((user, name), db).isEmpty, "Reusing database name with different server") - } - - def s3Bucket(user: AuthenticatedUser, s3Bucket: S3Bucket): Unit = { - s3Buckets.add((user, s3Bucket)) - } - - def secret(user: AuthenticatedUser, name: String, value: String): Unit = { - secrets.add((user, Secret(name, value))) - } - - def dropbox(user: AuthenticatedUser, dropboxToken: DropboxToken): Unit = { - dropboxTokens.put(user, dropboxToken) - } - - def oauth(user: AuthenticatedUser, name: String, token: NewHttpCredential): Unit = { - newHttpCreds.append((user, (name, token))) - } - - def credentials: CredentialsService = instance - - def setCredentials(credentials: CredentialsService): Unit = { - instance = credentials - } - - override def beforeAll(): Unit = { - super.beforeAll() - s3Buckets.foreach { case (user, bucket) => credentials.registerS3Bucket(user, bucket) } - rdbmsServers.foreach { case ((user, name), rdbms) => credentials.registerRDBMSServer(user, name, rdbms) } - newHttpCreds.foreach { case (user, (name, cred)) => credentials.registerNewHttpCredential(user, name, cred) } - dropboxTokens.foreach { case (user, token) => credentials.registerDropboxToken(user, token) } - secrets.foreach { case (user, secret) => credentials.registerSecret(user, secret) } - } - - override def afterAll(): Unit = { - s3Buckets.foreach { - case (user, bucket) => RawUtils.withSuppressNonFatalException(credentials.unregisterS3Bucket(user, bucket.name)) - } - rdbmsServers.foreach { - case ((user, name), _) => RawUtils.withSuppressNonFatalException(credentials.unregisterRDBMSServer(user, name)) - } - newHttpCreds.foreach { - case (user, (name, _)) => - RawUtils.withSuppressNonFatalException(credentials.unregisterNewHttpCredential(user, name)) - } - dropboxTokens.foreach { - case (user, _) => RawUtils.withSuppressNonFatalException(credentials.unregisterDropboxToken(user)) - } - secrets.foreach { - case (user, secret) => RawUtils.withSuppressNonFatalException(credentials.unregisterSecret(user, secret.name)) - } - setCredentials(null) - super.afterAll() - } - - protected def compare(expected: TokenCredential, actual: TokenCredential): Unit = { - assertResult(expected.provider)(actual.provider) - assertResult(expected.accessToken)(actual.accessToken) - assertResult(expected.scopes)(actual.scopes) - assertResult(expected.refreshToken)(actual.refreshToken) - assertResult(expected.expiresBy.map(_.truncatedTo(ChronoUnit.MILLIS)))( - actual.expiresBy.map(_.truncatedTo(ChronoUnit.MILLIS)) - ) - assertResult(expected.options)(actual.options) - } - - protected def compare(expected: ClientCredentialsCredential, actual: ClientCredentialsCredential): Unit = { - assertResult(expected.provider)(actual.provider) - assertResult(expected.clientId)(actual.clientId) - assertResult(expected.clientSecret)(actual.clientSecret) - assertResult(expected.options)(actual.options) - assertResult(expected.maybeAccessToken)(actual.maybeAccessToken) - assertResult(expected.maybeScopes)(actual.maybeScopes) - assertResult(expected.maybeExpiresBy.map(_.truncatedTo(ChronoUnit.MILLIS)))( - actual.maybeExpiresBy.map(_.truncatedTo(ChronoUnit.MILLIS)) - ) - } -} diff --git a/sources/src/test/scala/raw/creds/dropbox/DropboxTestCreds.scala b/sources/src/test/scala/raw/creds/dropbox/DropboxTestCreds.scala deleted file mode 100644 index 55831a503..000000000 --- a/sources/src/test/scala/raw/creds/dropbox/DropboxTestCreds.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.dropbox - -import raw.utils.Uid -import scala.util.Random -import raw.creds.api._ - -trait DropboxTestCreds { - - // For Raw Labs app - // Access token for "dropbox://Apps/Raw Test/" - val dropboxAccessToken = sys.env("RAW_DROPBOX_TEST_ACCESS_TOKEN") - val dropboxTokenUID = sys.env("RAW_DROPBOX_TEST_TOKEN_UID") - val dropboxToken = DropboxToken(dropboxAccessToken, "bearer", Uid(dropboxTokenUID)) - val dropboxClientId = sys.env("RAW_DROPBOX_TEST_CLIENT_ID") - - // The refresh token does not expire, but can be revoked - // https://www.dropbox.com/developers/documentation/http/documentation#auth-token-revoke - val expiredAccessToken = sys.env("RAW_EXPIRED_TEST_ACCESS_TOKEN") - val validUntil = "2022-02-08T14:45:35.959994891Z" - val scopes = Seq("scope1", "scope2") - val refreshToken = sys.env("RAW_REFRESH_TEST_TOKEN") - - // RAW Test Dropbox App - val dropboxRawTestAppClientId = sys.env("RAW_DROPBOX_RAW_TEST_APP_CLIENT_ID") - val dropboxRawTestAppClientSecret = sys.env("RAW_DROPBOX_RAW_TEST_APP_CLIENT_SECRET") - - // This token is expired. - val rawTestAccessToken = sys.env( - "RAW_DROPBOX_RAW_TEST_ACCESS_TOKEN" - ) - - // The refresh token should be valid indefinitely - val dropboxRefreshToken = sys.env("RAW_DROPBOX_TEST_REFRESH_TOKEN") - val dropboxBadRefreshToken = - dropboxRefreshToken.updated(Random.nextInt(dropboxRefreshToken.length), (Random.nextInt(26) + 'a').toChar) - val rawTestTokenUid = dropboxTokenUID - val rawTestTokenAccountId = sys.env("RAW_DROPBOX_TEST_ACCOUNT_ID") - - // The access token is expired, but the refresh token is valid forever - val dropboxRefreshTokenCredential = TokenCredential( - OAuth2Provider.Dropbox, - rawTestAccessToken, - None, - Some(scopes), - Some(dropboxRefreshToken), - Map("key1" -> "value1", "key2" -> "value2") - ) - - val dropboxBadRefreshTokenCredential = TokenCredential( - OAuth2Provider.Dropbox, - rawTestAccessToken, - None, - Some(scopes), - Some(dropboxBadRefreshToken), - Map("key1" -> "value1", "key2" -> "value2") - ) - - val dropboxLongLivedAccessToken = sys.env( - "RAW_DROPBOX_TEST_LONG_LIVED_ACCESS_TOKEN" - ) - // Long lived access token for "dropbox://Apps/Raw Test/" - // https://www.dropbox.com/developers/apps/info/97wyik3vzi5s5im - val dropboxAccessTokenCredential = TokenCredential( - OAuth2Provider.Dropbox, - dropboxLongLivedAccessToken, - None, - Some(scopes), - None, - Map("key1" -> "value1", "key2" -> "value2") - ) - - val dropboxBadLongLivedAccessToken = dropboxLongLivedAccessToken.updated( - Random.nextInt(dropboxLongLivedAccessToken.length), - (Random.nextInt(26) + 'a').toChar - ) - val dropboxBadAccessTokenCredential = TokenCredential( - OAuth2Provider.Dropbox, - dropboxBadLongLivedAccessToken, - None, - Some(scopes), - None, - Map("key1" -> "value1", "key2" -> "value2") - ) - - val bearerToken = BearerToken(dropboxAccessTokenCredential.accessToken, Map("key1" -> "value1", "key2" -> "value2")) -} diff --git a/sources/src/test/scala/raw/creds/http/OAuth2TestCreds.scala b/sources/src/test/scala/raw/creds/http/OAuth2TestCreds.scala deleted file mode 100644 index a026b5aa6..000000000 --- a/sources/src/test/scala/raw/creds/http/OAuth2TestCreds.scala +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.http - -import java.time.{LocalDateTime, ZoneOffset} -import scala.util.Random -import raw.creds.api._ - -trait OAuth2TestCreds { - - // RawTest, raw-test2 tenant - val auth0ClientId = sys.env("RAW_AUTH0_TEST_CLIENT_ID") - val auth0ClientSecret = sys.env("RAW_AUTH0_TEST_CLIENT_SECRET") - val auth0Domain = sys.env("RAW_AUTH0_TEST_DOMAIN") - val auth0MgmtApi = s"https://$auth0Domain/api/v2/" - - val auth0ClientCreds = ClientCredentialsCredential( - OAuth2Provider.Auth0, - auth0ClientId, - auth0ClientSecret, - Map("audience" -> auth0MgmtApi, "base_url" -> s"https://$auth0Domain") - ) - - //replace random char by another random char - val auth0BadClientId = auth0ClientId.updated(Random.nextInt(auth0ClientId.length), (Random.nextInt(26) + 'a').toChar) - val auth0BadClientIdCreds = ClientCredentialsCredential( - OAuth2Provider.Auth0, - auth0BadClientId, - auth0ClientSecret, - Map("audience" -> auth0MgmtApi, "base_url" -> s"https://$auth0Domain") - ) - - //replace random char by another random char - val auth0BadClientSecret = - auth0ClientSecret.updated(Random.nextInt(auth0ClientSecret.length), (Random.nextInt(26) + 'a').toChar) - val auth0BadClientSecretCreds = ClientCredentialsCredential( - OAuth2Provider.Auth0, - auth0ClientId, - auth0BadClientSecret, - Map("audience" -> auth0MgmtApi, "base_url" -> s"https://$auth0Domain") - ) - - val auth0AccessToken = TokenCredential(OAuth2Provider.Auth0, "not-supported", None, None, None, Map.empty) - - val shopifyAccessToken = sys.env("RAW_SHOPIFY_TEST_ACCESS_TOKEN") - val shopifyTokenCredential = TokenCredential(OAuth2Provider.Shopify, shopifyAccessToken, None, None, None, Map.empty) - - //replace random char by another random char - val shopifyBadAccessToken = - shopifyAccessToken.updated(Random.nextInt(shopifyAccessToken.length), (Random.nextInt(26) + 'a').toChar) - val shopifyBadTokenCredential = - TokenCredential(OAuth2Provider.Shopify, shopifyBadAccessToken, None, None, None, Map.empty) - - val shopifyClientCredential = ClientCredentialsCredential( - OAuth2Provider.Shopify, - "client-id-not-supported", - "client-secret-not-supported", - Map("key" -> "value-not-supported") - ) - - val twitterApiKey = sys.env("RAW_TWITTER_TEST_API_KEY") - val twiterApiSecret = sys.env("RAW_TWITTER_TEST_API_SECRET") - - val twitterClientCredential = ClientCredentialsCredential( - OAuth2Provider.Twitter, - twitterApiKey, - twiterApiSecret, - Map.empty - ) - - // Created on: 2022.06.02 (1654180772 UTC) - // Token expires: in 2 months (1659364775 UTC) - // Refresh token expires: in 12 months (1685716777 UTC) - val linkedInAccessToken = sys.env("RAW_LINKEDIN_TEST_ACCESS_TOKEN") - val linkedInRefreshToken = sys.env("RAW_LINKEDIN_TEST_REFRESH_TOKEN") - val linkedInClientId = sys.env("RAW_LINKEDIN_TEST_CLIENT_ID") - val linkedInClientSecret = sys.env("RAW_LINKEDIN_TEST_CLIENT_SECRET") - val linkedInTokenCredentials = TokenCredential( - OAuth2Provider.LinkedIn, - linkedInAccessToken, - Some(LocalDateTime.parse("2019-10-25T12:15:30").toInstant(ZoneOffset.UTC)), - None, - Some(linkedInRefreshToken), - Map( - "client_id" -> linkedInClientId, - "client_secret" -> linkedInClientSecret - ) - ) - // Non browser application created in zoho api console. - // flow: https://www.zoho.com/accounts/protocol/oauth/devices/client-protocol-flow.html - // validation: https://www.zoho.com/accounts/protocol/oauth/devices/initiation-request.html - val zohoAccessToken = sys.env("RAW_ZOHO_TEST_ACCESS_TOKEN") - val zohoRefreshToken = sys.env("RAW_ZOHO_TEST_REFRESH_TOKEN") - val zohoClientID = sys.env("RAW_ZOHO_TEST_CLIENT_ID") - val zohoClientSecret = sys.env("RAW_ZOHO_TEST_CLIENT_SECRET") - val zohoAccountsUrl = "https://accounts.zoho.com" - - val zohoTokenCredentials = TokenCredential( - OAuth2Provider.Zoho, - zohoAccessToken, - Some(LocalDateTime.parse("2019-10-25T12:15:30").toInstant(ZoneOffset.UTC)), - None, - Some(zohoRefreshToken), - Map( - "accounts_url" -> zohoAccountsUrl, - "client_id" -> zohoClientID, - "client_secret" -> zohoClientSecret - ) - ) - - val zohoBadAccessToken = TokenCredential( - OAuth2Provider.Zoho, - "bad-token", - Some(LocalDateTime.parse("2019-10-25T12:15:30").toInstant(ZoneOffset.UTC)), - None, - Some("bad-refresh-token"), - Map( - "accounts_url" -> zohoAccountsUrl, - "client_id" -> zohoClientID, - "client_secret" -> zohoClientSecret - ) - ) - - // Zoho client-Id/client-secret credentials are not supported - val zohoClientCredentials = ClientCredentialsCredential( - OAuth2Provider.Zoho, - clientId = zohoClientID, - clientSecret = zohoClientSecret, - options = Map("accounts_url" -> "https://accounts.zoho.com") - ) - - lazy val googleApiSecretKey = sys.env("RAW_GOOGLE_TEST_API_SECRET_KEY").stripMargin - - lazy val googleApiClientCredentials = ClientCredentialsCredential( - OAuth2Provider.GoogleApi, - clientId = sys.env("RAW_GOOGLE_TEST_API_CLIENT_ID"), - clientSecret = googleApiSecretKey, - options = Map( - "scope" -> "https://www.googleapis.com/auth/analytics.readonly", - "client_email" -> "raw-labs-analytics-serrvice@raw-labs-analytics.iam.gserviceaccount.com", - "token_uri" -> "https://oauth2.googleapis.com/token" - ) - ) -} diff --git a/sources/src/test/scala/raw/creds/jdbc/RDBMSTestCreds.scala b/sources/src/test/scala/raw/creds/jdbc/RDBMSTestCreds.scala deleted file mode 100644 index f58bbe150..000000000 --- a/sources/src/test/scala/raw/creds/jdbc/RDBMSTestCreds.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.jdbc - -import raw.creds.api._ -trait RDBMSTestCreds { - val mysqlTestHost = sys.env("RAW_MYSQL_TEST_HOST") - val mysqlTestDB = sys.env("RAW_MYSQL_TEST_DB") - val mysqlTestUser = sys.env("RAW_MYSQL_TEST_USER") - val mysqlTestPassword = sys.env("RAW_MYSQL_TEST_PASSWORD") - val mysqlCreds = MySqlCredential(mysqlTestHost, Some(3306), mysqlTestDB, Some(mysqlTestUser), Some(mysqlTestPassword)) - val pgsqlTestHost = sys.env("RAW_PGSQL_TEST_HOST") - val pgsqlTestDB = sys.env("RAW_PGSQL_TEST_DB") - val pgsqlTestUser = sys.env("RAW_PGSQL_TEST_USER") - val pgsqlTestPassword = sys.env("RAW_PGSQL_TEST_PASSWORD") - val pgsqlCreds = - PostgresqlCredential(pgsqlTestHost, Some(5432), pgsqlTestDB, Some(pgsqlTestUser), Some(pgsqlTestPassword)) - val oracleTestHost = sys.env("RAW_ORACLE_TEST_HOST") - val oracleTestDB = sys.env("RAW_ORACLE_TEST_DB") - val oracleTestUser = sys.env("RAW_ORACLE_TEST_USER") - val oracleTestPassword = sys.env("RAW_ORACLE_TEST_PASSWORD") - val oracleCreds = - OracleCredential(oracleTestHost, Some(1521), oracleTestDB, Some(oracleTestUser), Some(oracleTestPassword)) - val sqlServerTestHost = sys.env("RAW_SQLSERVER_TEST_HOST") - val sqlserverTestDB = sys.env("RAW_SQLSERVER_TEST_DB") - val sqlServerTestUser = sys.env("RAW_SQLSERVER_TEST_USER") - val sqlServerTestPassword = sys.env("RAW_SQLSERVER_TEST_PASSWORD") - val sqlServerCreds = SqlServerCredential( - sqlServerTestHost, - Some(1433), - sqlserverTestDB, - Some(sqlServerTestUser), - Some(sqlServerTestPassword) - ) - val teradataTestHost = sys.env("RAW_TERADATA_TEST_HOST") - val teradataTestUser = sys.env("RAW_TERADATA_TEST_USER") - val teradataTestPassword = sys.env("RAW_TERADATA_TEST_PASSWORD") - val teradataCreds = TeradataCredential(teradataTestHost, None, Some(teradataTestUser), Some(teradataTestPassword)) - val snowflakeTestHost = sys.env("RAW_SNOWFLAKE_TEST_HOST") - val snowflakeTestDB = sys.env("RAW_SNOWFLAKE_TEST_DB") - val snowflakeTestUser = sys.env("RAW_SNOWFLAKE_TEST_USER") - val snowflakeTestPassword = sys.env("RAW_SNOWFLAKE_TEST_PASSWORD") - val snowflakeCreds = SnowflakeCredential( - snowflakeTestHost, - snowflakeTestDB, - Some(snowflakeTestUser), - Some(snowflakeTestPassword), - Map("timezone" -> "UTC") - ) - val badMysqlCreds = - MySqlCredential("does-not-exist.raw-labs.com", Some(3306), "rdbmstest", Some("t0or"), Some("$up3r$3cr3tValu3")) -} diff --git a/sources/src/test/scala/raw/creds/local/LocalCredentialsTestContext.scala b/sources/src/test/scala/raw/creds/local/LocalCredentialsTestContext.scala deleted file mode 100644 index 4f8d3356c..000000000 --- a/sources/src/test/scala/raw/creds/local/LocalCredentialsTestContext.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.local - -import org.scalatest.BeforeAndAfterAll -import raw.utils.{RawTestSuite, RawUtils, SettingsTestContext} -import raw.creds.api.CredentialsTestContext - -trait LocalCredentialsTestContext extends BeforeAndAfterAll { - this: RawTestSuite with SettingsTestContext with CredentialsTestContext => - - protected var localCredentialsService: LocalCredentialsService = _ - - override def beforeAll(): Unit = { - super.beforeAll() - - property("raw.creds.impl", "local") - - localCredentialsService = new LocalCredentialsService() - setCredentials(localCredentialsService) - } - - override def afterAll(): Unit = { - if (localCredentialsService != null) { - RawUtils.withSuppressNonFatalException(localCredentialsService.stop()) - localCredentialsService = null - } - super.afterAll() - } -} diff --git a/sources/src/test/scala/raw/creds/oauth2/TestOauth2Clients.scala b/sources/src/test/scala/raw/creds/oauth2/TestOauth2Clients.scala deleted file mode 100644 index 64633edfa..000000000 --- a/sources/src/test/scala/raw/creds/oauth2/TestOauth2Clients.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.oauth2 - -import com.typesafe.scalalogging.StrictLogging -import org.scalatest.funsuite.AnyFunSuite -import raw.creds.api.{ClientCredentialsCredential, NewHttpCredential, TokenCredential} -import raw.creds.http.OAuth2TestCreds -import raw.creds.oauth2.api._ -import raw.creds.oauth2.auth0._ -import raw.creds.oauth2.linkedin._ -import raw.creds.oauth2.twitter._ -import raw.creds.oauth2.zoho._ -import raw.creds.oauth2.google._ -import raw.utils.SettingsTestContext - -class TestOauth2Clients extends AnyFunSuite with StrictLogging with SettingsTestContext with OAuth2TestCreds { - - def testTokenFromClient(client: OAuth2Client, credential: NewHttpCredential) = { - val token = credential match { - case ClientCredentialsCredential(_, clientId, clientSecret, options, _, _, _) => - client.newAccessTokenFromClientCredentials(clientId, clientSecret, options) - case TokenCredential(_, _, _, _, Some(refreshToken), options) => - client.newAccessTokenFromRefreshToken(refreshToken, options) - case TokenCredential(_, _, _, _, None, _) => - throw new AssertionError("refresh token has to be defined in token credential") - case _ => throw new AssertionError(s"unsupported credential type $credential") - } - logger.debug(s"token: ${token.accessToken}") - - assert(token.accessToken != "") - } - - test("auth0 client credentials") { - testTokenFromClient(new Auth0OAuth2Client, auth0ClientCreds) - } - - test("linkedin renew token") { - testTokenFromClient(new LinkedInOAuth2Client, linkedInTokenCredentials) - } - - test("twitter client credentials get token") { - testTokenFromClient(new TwitterOAuth2Client, twitterClientCredential) - } - - test("zoho refresh token") { - testTokenFromClient(new ZohoOAuth2Client, zohoTokenCredentials) - } - - test("google api credentials") { - testTokenFromClient(new GoogleApiKeyOAuth2Client, googleApiClientCredentials) - } -} diff --git a/sources/src/test/scala/raw/creds/s3/S3TestCreds.scala b/sources/src/test/scala/raw/creds/s3/S3TestCreds.scala deleted file mode 100644 index c01019600..000000000 --- a/sources/src/test/scala/raw/creds/s3/S3TestCreds.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.creds.s3 - -import raw.creds.api._ - -trait S3TestCreds { - - // Bucket with public access - val UnitTestPublicBucket = S3Bucket("rawlabs-public-test-data", Some("eu-west-1"), None) - val accessKeyId = sys.env("RAW_AWS_ACCESS_KEY_ID") - val secretKeyId = sys.env("RAW_AWS_SECRET_ACCESS_KEY") - // IAM user 'unit-test-private-bucket', which only has permissions only to access bucket 'rawlabs-private-test-data' - val UnitTestPrivateBucket = S3Bucket( - "rawlabs-private-test-data", - Some("eu-west-1"), - Some(AWSCredentials(accessKeyId, secretKeyId)) - ) - val UnitTestPrivateBucket2 = S3Bucket( - "rawlabs-unit-tests", - Some("eu-west-1"), - Some(AWSCredentials(accessKeyId, secretKeyId)) - ) - val UnitTestEmptyBucketPrivateBucket = S3Bucket( - "rawlabs-unit-test-empty-bucket", - Some("eu-west-1"), - Some(AWSCredentials(accessKeyId, secretKeyId)) - ) - - val UnitTestListRootPrivateBucket = S3Bucket( - "rawlabs-unit-test-list-root", - Some("eu-west-1"), - Some(AWSCredentials(accessKeyId, secretKeyId)) - ) - - val unitTestPrivateBucketUsEast1 = S3Bucket( - "rawlabs-unit-tests-us-east-1", - Some("us-east-1"), - Some(AWSCredentials(accessKeyId, secretKeyId)) - ) -} diff --git a/sources/src/test/scala/raw/sources/api/SourcesTestContext.scala b/sources/src/test/scala/raw/sources/api/SourcesTestContext.scala deleted file mode 100644 index 440521d89..000000000 --- a/sources/src/test/scala/raw/sources/api/SourcesTestContext.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.api - -import org.scalatest.BeforeAndAfterAll -import raw.utils.{RawTestSuite, SettingsTestContext} - -trait SourcesTestContext extends SettingsTestContext with BeforeAndAfterAll { - this: RawTestSuite => - -} diff --git a/sources/src/test/scala/raw/sources/bytestream/http/HttpLocationsTestContext.scala b/sources/src/test/scala/raw/sources/bytestream/http/HttpLocationsTestContext.scala index 93056393a..579bd01e7 100644 --- a/sources/src/test/scala/raw/sources/bytestream/http/HttpLocationsTestContext.scala +++ b/sources/src/test/scala/raw/sources/bytestream/http/HttpLocationsTestContext.scala @@ -10,7 +10,7 @@ * licenses/APL.txt. */ -package raw.sources.bytestream.api +package raw.sources.bytestream.http trait HttpLocationsTestContext { diff --git a/sources/src/test/scala/raw/sources/bytestream/http/TestHttpServer.scala b/sources/src/test/scala/raw/sources/bytestream/http/TestHttpServer.scala index abbef9c56..dc7604754 100644 --- a/sources/src/test/scala/raw/sources/bytestream/http/TestHttpServer.scala +++ b/sources/src/test/scala/raw/sources/bytestream/http/TestHttpServer.scala @@ -10,7 +10,7 @@ * licenses/APL.txt. */ -package raw.sources.bytestream.api +package raw.sources.bytestream.http import com.sun.net.httpserver.{Authenticator, BasicAuthenticator, HttpExchange, HttpHandler, HttpServer} import com.typesafe.scalalogging.StrictLogging diff --git a/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocation.scala b/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocation.scala deleted file mode 100644 index 8671294fb..000000000 --- a/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocation.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.in_memory - -import raw.utils.RawTestSuite -import raw.client.api._ -import raw.sources.api._ - -class TestInMemoryLocation extends RawTestSuite { - - test("in memory new location") { _ => - val settings = Map[LocationSettingKey, LocationSettingValue]( - ( - LocationSettingKey(InMemoryByteStreamLocation.codeDataKey), - LocationBinarySetting("hello world".getBytes(UTF_8().rawEncoding)) - ) - ) - val locationDescription = LocationDescription(InMemoryByteStreamLocation.schemaWithColon, settings) - val inMemoryLocation = new InMemoryByteStreamLocation(locationDescription) - assert(inMemoryLocation.getInputStream.readAllBytes().sameElements("hello world".getBytes(UTF_8().rawEncoding))) - assert(inMemoryLocation.rawUri.startsWith("in-memory")) - } - - test("in memory location errors") { _ => - val settings = Map[LocationSettingKey, LocationSettingValue]( - (LocationSettingKey(InMemoryByteStreamLocation.codeDataKey), LocationStringSetting("hello world")) - ) - val locationDescription = LocationDescription(InMemoryByteStreamLocation.schemaWithColon, settings) - val inMemoryLocation = new InMemoryByteStreamLocation(locationDescription) - assertThrows[MatchError](inMemoryLocation.getSeekableInputStream) - assertThrows[MatchError](inMemoryLocation.getInputStream) - assertThrows[AssertionError](inMemoryLocation.getLocalPath()) - } - -} diff --git a/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocationBuild.scala b/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocationBuild.scala deleted file mode 100644 index 1d3f87e18..000000000 --- a/sources/src/test/scala/raw/sources/bytestream/in_memory/TestInMemoryLocationBuild.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.bytestream.in_memory - -import raw.utils.RawTestSuite -import raw.sources.api._ -import raw.client.api._ - -class TestInMemoryLocationBuild extends RawTestSuite { - - test("in memory location builder") { _ => - val settings = Map[LocationSettingKey, LocationSettingValue]( - ( - LocationSettingKey(InMemoryByteStreamLocation.codeDataKey), - LocationBinarySetting("hello world".getBytes(UTF_8().rawEncoding)) - ) - ) - val locationDescription = LocationDescription(InMemoryByteStreamLocation.schemaWithColon, settings) - implicit val sourceContext: SourceContext = null - val inMemoryLocationBuilt = new InMemoryByteStreamLocationBuilder().build(locationDescription) - assert( - inMemoryLocationBuilt.getInputStream.readAllBytes().sameElements("hello world".getBytes(UTF_8().rawEncoding)) - ) - assert(inMemoryLocationBuilt.rawUri.startsWith("in-memory")) - } - - test("in memory location builder scheme") { _ => - val inMemoryLocationBuilder = new InMemoryByteStreamLocationBuilder() - assert(inMemoryLocationBuilder.schemes.size == 1) - assert(inMemoryLocationBuilder.schemes.head == InMemoryByteStreamLocation.schema) - } - - test("wrong schema build error") { _ => - val settings = Map[LocationSettingKey, LocationSettingValue]( - ( - LocationSettingKey(InMemoryByteStreamLocation.codeDataKey), - LocationBinarySetting("hello world".getBytes(UTF_8().rawEncoding)) - ) - ) - val locationDescription = LocationDescription("not-in-memory", settings) - implicit val sourceContext: SourceContext = null - - assertThrows[LocationException](new InMemoryByteStreamLocationBuilder().build(locationDescription)) - } - -} diff --git a/sources/src/test/scala/raw/sources/filesystem/dropbox/TestDropboxFileSystem.scala b/sources/src/test/scala/raw/sources/filesystem/dropbox/TestDropboxFileSystem.scala index d3dda3977..74e7f329c 100644 --- a/sources/src/test/scala/raw/sources/filesystem/dropbox/TestDropboxFileSystem.scala +++ b/sources/src/test/scala/raw/sources/filesystem/dropbox/TestDropboxFileSystem.scala @@ -12,25 +12,32 @@ package raw.sources.filesystem.dropbox +import com.dropbox.core.DbxRequestConfig +import com.dropbox.core.oauth.DbxCredential +import com.dropbox.core.v2.DbxClientV2 import raw.utils.RawTestSuite -import raw.creds.dropbox.DropboxTestCreds import raw.sources.filesystem.api.{FileSystem, TestFileSystems} import java.io.ByteArrayInputStream import scala.util.Try -class TestDropboxFileSystem extends RawTestSuite with TestFileSystems with DropboxTestCreds { +class TestDropboxFileSystem extends RawTestSuite with TestFileSystems { override val basePath = "/dropbox-test" - val dropboxClient = DropboxFileSystem.buildDbxClientV2(bearerToken) + val dropboxClient = new DropboxFileSystem( + new DbxClientV2( + DbxRequestConfig.newBuilder(settings.getString(BaseDropboxPath.DROPBOX_CLIENT_ID)).build(), + new DbxCredential(sys.env("RAW_DROPBOX_TEST_LONG_LIVED_ACCESS_TOKEN")) + ) + ) - override def newFileSystem: FileSystem = new DropboxFileSystem(bearerToken) + override def newFileSystem: FileSystem = dropboxClient override def writeTestFile(fs: FileSystem, parts: String*): Unit = { - Try(dropboxClient.files().createFolderV2(basePath)) + Try(dropboxClient.client.files().createFolderV2(basePath)) val in = new ByteArrayInputStream(Array[Byte]()) - Try(dropboxClient.files().uploadBuilder(buildPath(fs, parts.mkString(fs.fileSeparator))).uploadAndFinish(in)) + Try(dropboxClient.client.files().uploadBuilder(buildPath(fs, parts.mkString(fs.fileSeparator))).uploadAndFinish(in)) } } diff --git a/sources/src/test/scala/raw/sources/filesystem/dropbox/TestRootDropboxFileSystem.scala b/sources/src/test/scala/raw/sources/filesystem/dropbox/TestRootDropboxFileSystem.scala index bb5d862a3..49ffadb4a 100644 --- a/sources/src/test/scala/raw/sources/filesystem/dropbox/TestRootDropboxFileSystem.scala +++ b/sources/src/test/scala/raw/sources/filesystem/dropbox/TestRootDropboxFileSystem.scala @@ -12,14 +12,21 @@ package raw.sources.filesystem.dropbox +import com.dropbox.core.DbxRequestConfig +import com.dropbox.core.oauth.DbxCredential +import com.dropbox.core.v2.DbxClientV2 import com.typesafe.scalalogging.StrictLogging -import raw.creds.dropbox.DropboxTestCreds import raw.utils.{RawTestSuite, SettingsTestContext} -class TestRootDropboxFileSystem extends RawTestSuite with DropboxTestCreds with SettingsTestContext with StrictLogging { +class TestRootDropboxFileSystem extends RawTestSuite with SettingsTestContext with StrictLogging { test("list /") { _ => - val fs = new DropboxFileSystem(bearerToken) + val fs = new DropboxFileSystem( + new DbxClientV2( + DbxRequestConfig.newBuilder(settings.getString(BaseDropboxPath.DROPBOX_CLIENT_ID)).build(), + new DbxCredential(sys.env("RAW_DROPBOX_TEST_LONG_LIVED_ACCESS_TOKEN")) + ) + ) logger.debug("Result: " + fs.listContents("/").toList) assert(fs.listContents("/").nonEmpty) } diff --git a/sources/src/test/scala/raw/sources/filesystem/local/TestLocalFileSystem.scala b/sources/src/test/scala/raw/sources/filesystem/local/TestLocalFileSystem.scala index d4d323d07..8b9fc43a3 100644 --- a/sources/src/test/scala/raw/sources/filesystem/local/TestLocalFileSystem.scala +++ b/sources/src/test/scala/raw/sources/filesystem/local/TestLocalFileSystem.scala @@ -22,7 +22,7 @@ class TestLocalFileSystem extends RawTestSuite with TestFileSystems { override val basePath: String = Files.createTempDirectory("test-local").toFile.getAbsolutePath - override def newFileSystem: FileSystem = new LocalFileSystem + override def newFileSystem: FileSystem = LocalFileSystem override def writeTestFile(fs: FileSystem, parts: String*): Unit = { val f = new File(buildPath(fs, parts.mkString(fs.fileSeparator))) diff --git a/sources/src/test/scala/raw/sources/filesystem/s3/S3LocationsTestContext.scala b/sources/src/test/scala/raw/sources/filesystem/s3/S3LocationsTestContext.scala deleted file mode 100644 index cfd6c3883..000000000 --- a/sources/src/test/scala/raw/sources/filesystem/s3/S3LocationsTestContext.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.sources.filesystem.s3 - -import raw.creds.s3.S3TestCreds - -trait S3LocationsTestContext extends S3TestCreds { - - def s3Path(bucket: String, path: String): String = s"s3://$bucket/$path" - - lazy val publicationsHjsonGzipS3Directory = s3Path(UnitTestPrivateBucket.name, "publications/publications-hjson-gzip") - lazy val publicationsHjsonBzip2S3Directory = - s3Path(UnitTestPrivateBucket.name, "publications/publications-hjson-bzip2") - lazy val publicationsHjsonLz4S3Directory = s3Path(UnitTestPrivateBucket.name, "publications/publications-hjson-lz4") - lazy val publicationsHjsonDeflateS3Directory = - s3Path(UnitTestPrivateBucket.name, "publications/publications-hjson-deflate") - lazy val publicationsHjsonSnappyS3Directory = - s3Path(UnitTestPrivateBucket.name, "publications/publications-hjson-snappy") - lazy val publicationsHjsonBzip2S3File = s3Path(UnitTestPrivateBucket.name, "publications/publications.hjson.bz2") - lazy val publicationsHjsonGzS3File = s3Path(UnitTestPrivateBucket.name, "publications/publications.hjson.gz") - - lazy val publicationsHjsonS3 = s3Path(UnitTestPublicBucket.name, "publications/publications.hjson") - lazy val patientsParquetAvroS3 = s3Path(UnitTestPublicBucket.name, "patients.avro.parquet") - - // Public test data - lazy val studentsCsvS3 = s3Path(UnitTestPublicBucket.name, "students.csv") - lazy val authorsJsonS3 = s3Path(UnitTestPublicBucket.name, "publications/authors.json") - lazy val authorsHjsonS3 = s3Path(UnitTestPublicBucket.name, "publications/authors.hjson") - lazy val authorsParquetS3 = s3Path(UnitTestPublicBucket.name, "publications/authors.parquet") - lazy val textS3 = s3Path(UnitTestPublicBucket.name, "log_file.txt") - lazy val discogsS3 = s3Path(UnitTestPublicBucket.name, "discogs_2.xml") - - // Private test data, authenticated access - lazy val authorsHjsonPrivateS3 = s3Path(UnitTestPrivateBucket.name, "authors.hjson") - lazy val authorsJsonPrivateS3 = s3Path(UnitTestPrivateBucket.name, "authors.json") - lazy val studentsCsvPrivateS3 = s3Path(UnitTestPrivateBucket.name, "students.csv") - lazy val authorsParquetPrivateS3 = s3Path(UnitTestPrivateBucket.name, "authors.parquet") - - lazy val gerdGsstfXmlS3 = s3Path(UnitTestPublicBucket.name, "GSSTF.xml") - lazy val gerdGsstfXmlS3Wildcard = s3Path(UnitTestPublicBucket.name, "GSSTF*.xml") - - lazy val rd3652File = s3Path(UnitTestPublicBucket.name, "RD-3652.csv") - -} diff --git a/sources/src/test/scala/raw/sources/filesystem/s3/TestLargeDirectory.scala b/sources/src/test/scala/raw/sources/filesystem/s3/TestLargeDirectory.scala index 04dd0e47c..53573b820 100644 --- a/sources/src/test/scala/raw/sources/filesystem/s3/TestLargeDirectory.scala +++ b/sources/src/test/scala/raw/sources/filesystem/s3/TestLargeDirectory.scala @@ -14,17 +14,11 @@ package raw.sources.filesystem.s3 import com.typesafe.scalalogging.StrictLogging import org.scalatest.BeforeAndAfterAll -import raw.creds.s3.S3TestCreds import raw.utils.{RawTestSuite, SettingsTestContext} import scala.collection.mutable -class TestLargeDirectory - extends RawTestSuite - with BeforeAndAfterAll - with SettingsTestContext - with StrictLogging - with S3TestCreds { +class TestLargeDirectory extends RawTestSuite with BeforeAndAfterAll with SettingsTestContext with StrictLogging { val prefix = "large-folder" @@ -39,7 +33,12 @@ class TestLargeDirectory test("list large directory") { _ => val expected = genDataset(prefix) - val s3FileSystem = new S3FileSystem(UnitTestPrivateBucket) + val s3FileSystem = new S3FileSystem( + "rawlabs-private-test-data", + Some("eu-west-1"), + Some(sys.env("RAW_AWS_ACCESS_KEY_ID")), + Some(sys.env("RAW_AWS_SECRET_ACCESS_KEY")) + ) val iterator = s3FileSystem.listContentsWithMetadata(prefix) val actual = new mutable.HashSet[String]() for ((file, md) <- iterator) { diff --git a/sources/src/test/scala/raw/sources/filesystem/s3/TestRootS3FileSystem.scala b/sources/src/test/scala/raw/sources/filesystem/s3/TestRootS3FileSystem.scala index 100662908..6ab049e6d 100644 --- a/sources/src/test/scala/raw/sources/filesystem/s3/TestRootS3FileSystem.scala +++ b/sources/src/test/scala/raw/sources/filesystem/s3/TestRootS3FileSystem.scala @@ -14,41 +14,54 @@ package raw.sources.filesystem.s3 import com.typesafe.scalalogging.StrictLogging import raw.utils.{RawTestSuite, SettingsTestContext} -import raw.creds.api.S3Bucket -import raw.creds.s3.S3TestCreds -class TestRootS3FileSystem extends RawTestSuite with S3TestCreds with SettingsTestContext with StrictLogging { +class TestRootS3FileSystem extends RawTestSuite with SettingsTestContext with StrictLogging { test("list ''") { _ => - val fs = new S3FileSystem(UnitTestPrivateBucket) + val fs = new S3FileSystem( + "rawlabs-private-test-data", + Some("eu-west-1"), + Some(sys.env("RAW_AWS_ACCESS_KEY_ID")), + Some(sys.env("RAW_AWS_SECRET_ACCESS_KEY")) + ) val list = fs.listContents("").toList logger.debug("Result: " + list) } test("list bucket region us-east-1") { _ => - val fs = new S3FileSystem(unitTestPrivateBucketUsEast1) + val fs = new S3FileSystem( + "rawlabs-unit-tests-us-east-1", + Some("us-east-1"), + Some(sys.env("RAW_AWS_ACCESS_KEY_ID")), + Some(sys.env("RAW_AWS_SECRET_ACCESS_KEY")) + ) val list = fs.listContents("").toList logger.debug("Result: " + list) } test("list bucket us-east-1 without specifying a region") { _ => - val bucket = S3Bucket(unitTestPrivateBucketUsEast1.name, None, unitTestPrivateBucketUsEast1.credentials) - val fs = new S3FileSystem(bucket) + val fs = new S3FileSystem( + "rawlabs-unit-tests-us-east-1", + None, + Some(sys.env("RAW_AWS_ACCESS_KEY_ID")), + Some(sys.env("RAW_AWS_SECRET_ACCESS_KEY")) + ) val list = fs.listContents("").toList logger.debug("Result: " + list) } } -class TestRootOfEmptyBucketS3FileSystem - extends RawTestSuite - with S3TestCreds - with SettingsTestContext - with StrictLogging { +class TestRootOfEmptyBucketS3FileSystem extends RawTestSuite with SettingsTestContext with StrictLogging { test("list ''") { _ => - val fs = new S3FileSystem(UnitTestEmptyBucketPrivateBucket) + val fs = new S3FileSystem( + "rawlabs-unit-test-empty-bucket", + Some("eu-west-1"), + Some(sys.env("RAW_AWS_ACCESS_KEY_ID")), + Some(sys.env("RAW_AWS_SECRET_ACCESS_KEY")) + ) val list = fs.listContents("").toList assert(list.isEmpty) } diff --git a/sources/src/test/scala/raw/sources/filesystem/s3/TestS3FileSystem.scala b/sources/src/test/scala/raw/sources/filesystem/s3/TestS3FileSystem.scala index 77f3fbb7a..1d8392c1f 100644 --- a/sources/src/test/scala/raw/sources/filesystem/s3/TestS3FileSystem.scala +++ b/sources/src/test/scala/raw/sources/filesystem/s3/TestS3FileSystem.scala @@ -13,8 +13,6 @@ package raw.sources.filesystem.s3 import raw.utils.RawTestSuite -import raw.creds.api.S3Bucket -import raw.creds.s3.S3TestCreds import raw.sources.filesystem.api.{FileSystem, TestFileSystems} import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.core.exception.SdkClientException @@ -25,25 +23,32 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest import java.nio.charset.StandardCharsets -trait TestS3FileSystem extends TestFileSystems with S3TestCreds { +trait TestS3FileSystem extends TestFileSystems { this: RawTestSuite => - def bucket: S3Bucket = UnitTestPrivateBucket + val bucketName: String + + val bucketRegion: String + + val bucketAccessKey: String + + val bucketSecretKey: String lazy val awsClient = { val credentials = AwsBasicCredentials.create( - bucket.credentials.get.accessKey, - bucket.credentials.get.secretKey + bucketAccessKey, + bucketSecretKey ) S3Client .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .region(Region.of(bucket.region.get)) + .region(Region.of(bucketRegion)) .build() } - override val newFileSystem: FileSystem = new S3FileSystem(bucket) + override lazy val newFileSystem: FileSystem = + new S3FileSystem(bucketName, Some(bucketRegion), Some(bucketAccessKey), Some(bucketSecretKey)) override def writeTestFile(fs: FileSystem, parts: String*): Unit = { val s3Path = buildPath(fs, parts.mkString(fs.fileSeparator)) @@ -54,7 +59,7 @@ trait TestS3FileSystem extends TestFileSystems with S3TestCreds { try { val putRequest = PutObjectRequest .builder() - .bucket(bucket.name) + .bucket(bucketName) .key(s3Path) .build() @@ -79,21 +84,32 @@ trait TestS3FileSystem extends TestFileSystems with S3TestCreds { } class TestForwardSlashS3FileSystem extends RawTestSuite with TestS3FileSystem { + override val bucketName = "rawlabs-private-test-data" + override val bucketRegion = "eu-west-1" + override val bucketAccessKey = sys.env("RAW_AWS_ACCESS_KEY_ID") + override val bucketSecretKey = sys.env("RAW_AWS_SECRET_ACCESS_KEY") override val basePath = "/s3-test" override def filterResults(p: String): Boolean = !p.startsWith(s"${basePath.stripPrefix("/")}/tmp-") } class TestNoForwardSlashS3FileSystem extends RawTestSuite with TestS3FileSystem { + override val bucketName = "rawlabs-private-test-data" + override val bucketRegion = "eu-west-1" + override val bucketAccessKey = sys.env("RAW_AWS_ACCESS_KEY_ID") + override val bucketSecretKey = sys.env("RAW_AWS_SECRET_ACCESS_KEY") override val basePath = "s3-test" override def filterResults(p: String): Boolean = !p.startsWith(s"$basePath/tmp-") } class TestRootOfBucketS3FileSystem extends RawTestSuite with TestS3FileSystem { + override val bucketName = "rawlabs-unit-tests\"" + override val bucketRegion = "eu-west-1" + override val bucketAccessKey = sys.env("RAW_AWS_ACCESS_KEY_ID") + override val bucketSecretKey = sys.env("RAW_AWS_SECRET_ACCESS_KEY") + override val basePath = "" override def filterResults(p: String): Boolean = !p.startsWith("tmp-") - override def bucket: S3Bucket = UnitTestPrivateBucket2 - override def buildPath(fs: FileSystem, relativePath: String): String = relativePath } diff --git a/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala b/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala index 4027d0ed3..8120048d5 100644 --- a/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala +++ b/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala @@ -31,7 +31,7 @@ import scala.util.control.NonFatal */ class SqlCompilerService()(implicit protected val settings: RawSettings) extends CompilerService { - private val connectionPool = new SqlConnectionPool() + private val connectionPool = new SqlConnectionPool // A short lived database metadata (schema/table/column names) indexed by JDBC URL. private val metadataBrowsers = { diff --git a/sql-client/src/test/scala/raw/client/sql/TestSqlCompilerServiceAirports.scala b/sql-client/src/test/scala/raw/client/sql/TestSqlCompilerServiceAirports.scala index 832400c82..abb545854 100644 --- a/sql-client/src/test/scala/raw/client/sql/TestSqlCompilerServiceAirports.scala +++ b/sql-client/src/test/scala/raw/client/sql/TestSqlCompilerServiceAirports.scala @@ -36,7 +36,7 @@ class TestSqlCompilerServiceAirports private var jdbcUrl: String = _ // Username equals the database - private var user: InteractiveUser = _ + private var user: RawUid = _ override def beforeAll(): Unit = { super.beforeAll() @@ -49,7 +49,7 @@ class TestSqlCompilerServiceAirports val stmt = conn.createStatement() stmt.execute(sql) - user = InteractiveUser(Uid(container.databaseName), "fdw user", "email", Seq.empty) + user = RawUid(container.databaseName) jdbcUrl = { val dbPort = container.mappedPort(5432).toString @@ -78,6 +78,10 @@ class TestSqlCompilerServiceAirports user, if (params.isEmpty) None else Some(params.toArray), scopes, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl) ) @@ -87,6 +91,10 @@ class TestSqlCompilerServiceAirports user, if (params.isEmpty) None else Some(params.toArray), scopes, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "csv"), jdbcUrl = Some(jdbcUrl) ) diff --git a/sql-client/src/test/scala/raw/client/sql/TestSqlConnectionFailures.scala b/sql-client/src/test/scala/raw/client/sql/TestSqlConnectionFailures.scala index da9d47246..7627d6250 100644 --- a/sql-client/src/test/scala/raw/client/sql/TestSqlConnectionFailures.scala +++ b/sql-client/src/test/scala/raw/client/sql/TestSqlConnectionFailures.scala @@ -48,9 +48,9 @@ class TestSqlConnectionFailures ) Class.forName("org.postgresql.Driver") - private var users: Set[InteractiveUser] = _ + private var users: Set[RawUid] = _ - private def jdbcUrl(user: AuthenticatedUser) = { + private def jdbcUrl(user: RawUid) = { val dbPort = container.mappedPort(5432).toString val dbName = user.uid val username = container.username @@ -63,7 +63,7 @@ class TestSqlConnectionFailures // For each user we create a specific database and load the example schema. users = { - val items = for (i <- 1 to nUsers) yield InteractiveUser(Uid(s"db$i"), "fdw user", "email", Seq.empty) + val items = for (i <- 1 to nUsers) yield RawUid(s"db$i") items.toSet } @@ -80,7 +80,7 @@ class TestSqlConnectionFailures try { val stmt = conn.createStatement() for (user <- users) { - val r = stmt.executeUpdate(s"CREATE DATABASE ${user.uid.uid}") + val r = stmt.executeUpdate(s"CREATE DATABASE ${user.uid}") assert(r == 0) } } finally { @@ -466,7 +466,7 @@ class TestSqlConnectionFailures private def runExecute( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String, arg: Int ): ExecutionResponse = { @@ -474,6 +474,10 @@ class TestSqlConnectionFailures user, Some(Array("arg" -> RawInt(arg))), Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) @@ -487,7 +491,7 @@ class TestSqlConnectionFailures private def runHover( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String, pos: Pos ): HoverResponse = { @@ -495,6 +499,10 @@ class TestSqlConnectionFailures user, None, Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) @@ -503,7 +511,7 @@ class TestSqlConnectionFailures private def runWordCompletion( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String, prefix: String, pos: Pos @@ -512,6 +520,10 @@ class TestSqlConnectionFailures user, None, Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) @@ -520,7 +532,7 @@ class TestSqlConnectionFailures private def runDotCompletion( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String, pos: Pos ): AutoCompleteResponse = { @@ -528,6 +540,10 @@ class TestSqlConnectionFailures user, None, Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) @@ -536,13 +552,17 @@ class TestSqlConnectionFailures private def runGetProgramDescription( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String ): GetProgramDescriptionResponse = { val env = ProgramEnvironment( user, None, Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) @@ -551,13 +571,17 @@ class TestSqlConnectionFailures private def runValidate( compilerService: CompilerService, - user: AuthenticatedUser, + user: RawUid, code: String ): ValidateResponse = { val env = ProgramEnvironment( user, None, Set.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, Map("output-format" -> "json"), jdbcUrl = Some(jdbcUrl(user)) ) diff --git a/utils/src/main/scala/raw/utils/AuthenticatedUser.scala b/utils/src/main/scala/raw/utils/AuthenticatedUser.scala deleted file mode 100644 index feac2294e..000000000 --- a/utils/src/main/scala/raw/utils/AuthenticatedUser.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2023 RAW Labs S.A. - * - * Use of this software is governed by the Business Source License - * included in the file licenses/BSL.txt. - * - * As of the Change Date specified in that file, in accordance with - * the Business Source License, use of this software will be governed - * by the Apache License, Version 2.0, included in the file - * licenses/APL.txt. - */ - -package raw.utils - -import com.fasterxml.jackson.annotation.JsonSubTypes.{Type => JsonType} -import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes( - Array( - new JsonType(value = classOf[InteractiveUser], name = "interactive"), - new JsonType(value = classOf[ApiUser], name = "api"), - new JsonType(value = classOf[ImpersonatedUser], name = "impersonated") - ) -) -sealed trait AuthenticatedUser { - def uid: Uid - def name: String - def permissions: Seq[String] -} - -final case class InteractiveUser(uid: Uid, name: String, email: String, permissions: Seq[String] = Seq.empty) - extends AuthenticatedUser - -final case class ApiUser( - uid: Uid, - name: String, - audience: Seq[String], - issuer: String, - permissions: Seq[String], - scopes: Seq[String] -) extends AuthenticatedUser - -final case class ImpersonatedUser( - uid: Uid, - name: String, - permissions: Seq[String], - impersonatedUser: AuthenticatedUser, - impersonater: AuthenticatedUser -) extends AuthenticatedUser - -object AuthenticatedUser { - - private val mapper: ObjectMapper with ClassTagExtensions = { - val om = new ObjectMapper() with ClassTagExtensions - om.registerModule(DefaultScalaModule) - om - } - - private val reader = mapper.readerFor[AuthenticatedUser] - private val writer = mapper.writerFor[AuthenticatedUser] - - def toString(user: AuthenticatedUser): String = { - writer.writeValueAsString(user) - } - - def fromString(str: String): AuthenticatedUser = { - reader.readValue(str) - } - -} diff --git a/utils/src/main/scala/raw/utils/Uid.scala b/utils/src/main/scala/raw/utils/RawUid.scala similarity index 89% rename from utils/src/main/scala/raw/utils/Uid.scala rename to utils/src/main/scala/raw/utils/RawUid.scala index ae943e0a3..50d1f0b00 100644 --- a/utils/src/main/scala/raw/utils/Uid.scala +++ b/utils/src/main/scala/raw/utils/RawUid.scala @@ -12,6 +12,6 @@ package raw.utils -final case class Uid(uid: String) extends AnyVal { +final case class RawUid(uid: String) { override def toString: String = uid }