From 39211c66db8b9e64524c5f1657cbb88c09ee5e7c Mon Sep 17 00:00:00 2001 From: Benjamin Gaidioz Date: Thu, 11 Jan 2024 10:37:40 +0100 Subject: [PATCH] working but with bad tests --- sql-client/build.sbt | 2 +- .../raw/client/sql/SqlCompilerService.scala | 20 ++- .../sql/writers/TypedResultSetCsvWriter.scala | 115 ++++++++++++++++++ .../writers/TypedResultSetJsonWriter.scala | 103 ++++++++++++++++ 4 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetCsvWriter.scala create mode 100644 sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetJsonWriter.scala diff --git a/sql-client/build.sbt b/sql-client/build.sbt index 6999257ba..68a4aa817 100644 --- a/sql-client/build.sbt +++ b/sql-client/build.sbt @@ -146,6 +146,6 @@ publishLocal := (publishLocal dependsOn Def.sequential(outputVersion, publishM2) libraryDependencies ++= Seq( rawClient % "compile->compile;test->test", postgresqlDeps, - hikariCP) ++ truffleCompiler + hikariCP) Compile / packageBin / packageOptions += Package.ManifestAttributes("Automatic-Module-Name" -> "raw.sql.client") diff --git a/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala b/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala index 56f4b8987..92dfe110b 100644 --- a/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala +++ b/sql-client/src/main/scala/raw/client/sql/SqlCompilerService.scala @@ -13,13 +13,12 @@ package raw.client.sql import com.google.common.cache.{CacheBuilder, CacheLoader} -import org.graalvm.polyglot.{Context, HostAccess, Value} import raw.client.api._ -import raw.client.writers.{TypedPolyglotCsvWriter, TypedPolyglotJsonWriter} +import raw.client.sql.writers.{TypedResultSetCsvWriter, TypedResultSetJsonWriter} import raw.utils.{AuthenticatedUser, RawSettings, RawUtils} import java.io.{IOException, OutputStream} -import java.sql.{SQLException, SQLTimeoutException} +import java.sql.{ResultSet, SQLException, SQLTimeoutException} import scala.collection.mutable class SqlCompilerService(maybeClassLoader: Option[ClassLoader] = None)(implicit protected val settings: RawSettings) @@ -106,12 +105,9 @@ class SqlCompilerService(maybeClassLoader: Option[ClassLoader] = None)(implicit case Right(info) => try { val tipe = info.outputType - val access = HostAccess.newBuilder().allowMapAccess(true).allowIteratorAccess(true).build() - val ctx = Context.newBuilder().allowHostAccess(access).build() environment.maybeArguments.foreach(array => setParams(pstmt, array)) val r = pstmt.executeQuery() - val v = ctx.asValue(new ResultSetIterator(r, ctx)) - render(environment, tipe, v, outputStream) + render(environment, tipe, r, outputStream) } catch { case e: SQLException => ExecutionRuntimeFailure(e.getMessage) } @@ -134,14 +130,14 @@ class SqlCompilerService(maybeClassLoader: Option[ClassLoader] = None)(implicit private def render( environment: ProgramEnvironment, tipe: RawType, - v: Value, + v: ResultSet, outputStream: OutputStream ): ExecutionResponse = { environment.options .get("output-format") .map(_.toLowerCase) match { case Some("csv") => - if (!TypedPolyglotCsvWriter.outputWriteSupport(tipe)) { + if (!TypedResultSetCsvWriter.outputWriteSupport(tipe)) { ExecutionRuntimeFailure("unsupported type") } val windowsLineEnding = environment.options.get("windows-line-ending") match { @@ -149,7 +145,7 @@ class SqlCompilerService(maybeClassLoader: Option[ClassLoader] = None)(implicit case _ => false //settings.config.getBoolean("raw.compiler.windows-line-ending") } val lineSeparator = if (windowsLineEnding) "\r\n" else "\n" - val csvWriter = new TypedPolyglotCsvWriter(outputStream, lineSeparator) + val csvWriter = new TypedResultSetCsvWriter(outputStream, lineSeparator) try { csvWriter.write(v, tipe) ExecutionSuccess @@ -159,10 +155,10 @@ class SqlCompilerService(maybeClassLoader: Option[ClassLoader] = None)(implicit RawUtils.withSuppressNonFatalException(csvWriter.close()) } case Some("json") => - if (!TypedPolyglotJsonWriter.outputWriteSupport(tipe)) { + if (!TypedResultSetJsonWriter.outputWriteSupport(tipe)) { ExecutionRuntimeFailure("unsupported type") } - val w = new TypedPolyglotJsonWriter(outputStream) + val w = new TypedResultSetJsonWriter(outputStream) try { w.write(v, tipe) ExecutionSuccess diff --git a/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetCsvWriter.scala b/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetCsvWriter.scala new file mode 100644 index 000000000..6c2d4b66e --- /dev/null +++ b/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetCsvWriter.scala @@ -0,0 +1,115 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.client.sql.writers + +import com.fasterxml.jackson.core.{JsonEncoding, JsonParser} +import com.fasterxml.jackson.dataformat.csv.CsvGenerator.Feature.STRICT_CHECK_FOR_QUOTING +import com.fasterxml.jackson.dataformat.csv.{CsvFactory, CsvSchema} +import raw.client.api._ +import raw.utils.RecordFieldsNaming + +import java.io.{IOException, OutputStream} +import java.sql.ResultSet +import java.time.format.DateTimeFormatter +import scala.annotation.tailrec + +object TypedResultSetCsvWriter { + + def outputWriteSupport(tipe: RawType): Boolean = tipe match { + case _: RawIterableType => true + case _: RawListType => true + case _ => false + } + +} + +class TypedResultSetCsvWriter(os: OutputStream, lineSeparator: String) { + + final private val gen = + try { + val factory = new CsvFactory + factory.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE) // Don't close file descriptors automatically + factory.createGenerator(os, JsonEncoding.UTF8) + } catch { + case e: IOException => throw new RuntimeException(e) + } + + private val schemaBuilder = CsvSchema.builder() + schemaBuilder.setColumnSeparator(',') + schemaBuilder.setUseHeader(true) + schemaBuilder.setLineSeparator(lineSeparator) + schemaBuilder.setQuoteChar('"') + schemaBuilder.setNullValue("") + + final private val dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd") + final private val timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS") + final private val timeFormatterNoMs = DateTimeFormatter.ofPattern("HH:mm:ss") + final private val timestampFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS") + final private val timestampFormatterNoMs = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss") + + @throws[IOException] + def write(resultSet: ResultSet, t: RawType): Unit = { + val RawIterableType(RawRecordType(atts, _, _), _, _) = t + val keys = new java.util.Vector[String] + atts.foreach(a => keys.add(a.idn)) + val distincted = RecordFieldsNaming.makeDistinct(keys) + val columnNames = atts.map(_.idn) + for (colName <- columnNames) { + schemaBuilder.addColumn(colName) + } + gen.setSchema(schemaBuilder.build) + gen.enable(STRICT_CHECK_FOR_QUOTING) + while (resultSet.next()) { + for (i <- 0 until distincted.size()) { + writeValue(resultSet, i + 1, atts(i).tipe) + } + } + } + + @throws[IOException] + @tailrec + private def writeValue(v: ResultSet, i: Int, t: RawType): Unit = { + if (t.nullable) { + if (v == null) gen.writeNull() + else writeValue(v, i, t.cloneNotNullable) + } else t match { + case _: RawBoolType => gen.writeBoolean(v.getBoolean(i)) + case _: RawByteType => gen.writeNumber(v.getByte(i).toInt) + case _: RawShortType => gen.writeNumber(v.getShort(i).toInt) + case _: RawIntType => gen.writeNumber(v.getInt(i)) + case _: RawLongType => gen.writeNumber(v.getLong(i)) + case _: RawFloatType => gen.writeNumber(v.getFloat(i)) + case _: RawDoubleType => gen.writeNumber(v.getDouble(i)) + case _: RawDecimalType => gen.writeString(v.getBigDecimal(i).toString) + case _: RawStringType => gen.writeString(v.getString(i)) + case _: RawDateType => + val date = v.getDate(i).toLocalDate + gen.writeString(dateFormatter.format(date)) + case _: RawTimeType => + val time = v.getTime(i).toLocalTime + val formatter = if (time.getNano > 0) timeFormatter else timeFormatterNoMs + val formatted = formatter.format(time) + gen.writeString(formatted) + case _: RawTimestampType => + val dateTime = v.getTimestamp(i).toLocalDateTime + val formatter = if (dateTime.getNano > 0) timestampFormatter else timestampFormatterNoMs + val formatted = formatter.format(dateTime) + gen.writeString(formatted) + case _ => throw new RuntimeException("unsupported type") + } + } + + def close(): Unit = { + gen.close() + } +} diff --git a/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetJsonWriter.scala b/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetJsonWriter.scala new file mode 100644 index 000000000..a9e81032f --- /dev/null +++ b/sql-client/src/main/scala/raw/client/sql/writers/TypedResultSetJsonWriter.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package raw.client.sql.writers + +import com.fasterxml.jackson.core.{JsonEncoding, JsonFactory, JsonParser} +import raw.client.api._ +import raw.utils.RecordFieldsNaming + +import java.io.{IOException, OutputStream} +import java.sql.ResultSet +import java.time.format.DateTimeFormatter +import scala.annotation.tailrec + +object TypedResultSetJsonWriter { + + def outputWriteSupport(tipe: RawType): Boolean = tipe match { + case _: RawIterableType => true + case _: RawListType => true + case _ => false + } + +} + +class TypedResultSetJsonWriter(os: OutputStream) { + + final private val gen = + try { + val factory = new JsonFactory + factory.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE) // Don't close file descriptors automatically + factory.createGenerator(os, JsonEncoding.UTF8) + } catch { + case e: IOException => throw new RuntimeException(e) + } + + final private val dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd") + final private val timeFormatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS") + final private val timestampFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS") + + @throws[IOException] + def write(resultSet: ResultSet, t: RawType): Unit = { + val RawIterableType(RawRecordType(atts, _, _), _, _) = t + val keys = new java.util.Vector[String] + atts.foreach(a => keys.add(a.idn)) + val distincted = RecordFieldsNaming.makeDistinct(keys) + gen.writeStartArray() + while (resultSet.next()) { + gen.writeStartObject() + for (i <- 0 until distincted.size()) { + val field = distincted.get(i) + val t = atts(i).tipe + gen.writeFieldName(field) + writeValue(resultSet, i + 1, t) + } + gen.writeEndObject() + } + gen.writeEndArray() + } + + @throws[IOException] + @tailrec + private def writeValue(v: ResultSet, i: Int, t: RawType): Unit = { + if (t.nullable) { + if (v == null) gen.writeNull() + else writeValue(v, i, t.cloneNotNullable) + } else t match { + case _: RawBoolType => gen.writeBoolean(v.getBoolean(i)) + case _: RawByteType => gen.writeNumber(v.getByte(i).toInt) + case _: RawShortType => gen.writeNumber(v.getShort(i).toInt) + case _: RawIntType => gen.writeNumber(v.getInt(i)) + case _: RawLongType => gen.writeNumber(v.getLong(i)) + case _: RawFloatType => gen.writeNumber(v.getFloat(i)) + case _: RawDoubleType => gen.writeNumber(v.getDouble(i)) + case _: RawDecimalType => gen.writeString(v.getBigDecimal(i).toString) + case _: RawStringType => gen.writeString(v.getString(i)) + case _: RawDateType => + val date = v.getDate(i).toLocalDate + gen.writeString(dateFormatter.format(date)) + case _: RawTimeType => + val time = v.getTime(i).toLocalTime + val formatted = timeFormatter.format(time) + gen.writeString(formatted) + case _: RawTimestampType => + val dateTime = v.getTimestamp(i).toLocalDateTime + val formatted = timestampFormatter.format(dateTime) + gen.writeString(formatted) + case _ => throw new RuntimeException("unsupported type") + } + } + + def close(): Unit = { + gen.close() + } +}