diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..464beac --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,33 @@ +name: CI +on: + pull_request: + paths: + - .github/workflows/ci.yaml + - .sbtopts + - build.sbt + - .scalafmt.conf + - project/** + - src/** + +defaults: + run: + shell: bash + +env: + GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }} + +jobs: + code-check: + runs-on: self-hosted + container: + image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19 + steps: + - uses: actions/checkout@v4 + - run: sbt headerCheckAll + test: + runs-on: self-hosted + container: + image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19 + steps: + - uses: actions/checkout@v4 + - run: sbt clean test diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..85b540c --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,22 @@ +name: publish +on: + push: + tags: + - "v*.*.*" + +env: + GITHUB_TOKEN: ${{ secrets.WRITE_PACKAGES }} + +jobs: + publish: + runs-on: self-hosted + container: + image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: sbt publish + run: | + git config --global --add safe.directory $GITHUB_WORKSPACE + sbt clean publish diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6fdef46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.class +.metals/ +.git/ +.vscode/ +target/ +.bsp/ +.idea/ diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000..71adad5 --- /dev/null +++ b/.sbtopts @@ -0,0 +1,7 @@ +-J-Xmx4G +-J-Xms1G +-J-Xss4M +-J-XX:+UseG1GC +-J--add-opens=java.base/java.lang=ALL-UNNAMED +-J--add-opens=java.base/java.util=ALL-UNNAMED +-J-XX:+CrashOnOutOfMemoryError diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 0000000..9c71391 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,136 @@ +version = 2.7.5 +maxColumn = 120 + +optIn.forceBlankLineBeforeDocstring = true +trailingCommas = "never" + +align { + openParenCallSite = false + openParenDefnSite = false + preset=some + stripMargin = false + tokens = [] +} + +assumeStandardLibraryStripMargin = true +binPack.parentConstructors = "never" + +continuationIndent { + callSite = 2 + ctorSite = 4 + defnSite = 4 + extendSite = 4 +} + +danglingParentheses { + ctrlSite = true + defnSite = true +} + + +docstrings.style = "Asterisk" // JavaDoc +includeCurlyBraceInSelectChains = true +includeNoParensInSelectChains = false + +literals { + double = "Lower" + float = "Lower" + hexDigits = "Lower" + hexPrefix = "Lower" + long = "Upper" +} + +newlines { + afterCurlyLambdaParams = squash + afterCurlyLambdaParams=squash + alwaysBeforeElseAfterCurlyIf = false + beforeCurlyLambdaParams = multilineWithCaseOnly + beforeMultiline = fold + beforeMultilineDef = false + implicitParamListModifierPrefer = before +} + +rewrite { + redundantBraces { + generalExpressions = false + ifElseExpressions = false + methodBodies = false + parensForOneLineApply = true + stringInterpolation = true + } + rules = [ + RedundantBraces, + RedundantParens, + SortModifiers, + SortImports + ] + redundantBraces { + generalExpressions = false + methodBodies = false + ifElseExpressions = false + parensForOneLineApply = true + stringInterpolation = true + } + sortModifiers { + order = [ + "implicit", + "final", + "sealed", + "abstract", + "override", + "private", + "protected", + "lazy" + ] + } +} + +continuationIndent { + callSite = 2 + defnSite = 4 + ctorSite = 4 + extendSite = 4 +} + +align { + preset=some + tokens = [] + openParenCallSite = false + openParenDefnSite = false + preset = some + stripMargin = false + tokens = [] +} + +danglingParentheses { + defnSite = true + ctrlSite = true +} + +spaces { + afterKeywordBeforeParen = true + afterSymbolicDefs = false + beforeContextBoundColon = "Never" + inByNameTypes = true + inParentheses = false + neverAroundInfixTypes = [] + afterKeywordBeforeParen = true + inByNameTypes = true + afterSymbolicDefs = false +} + +literals { + long = "Upper" + float = "Lower" + double = "Lower" + hexPrefix = "Lower" + hexDigits = "Lower" +} + +docstrings { + style = "Asterisk" // JavaDoc +} + +binPack { + parentConstructors = "Never" +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..262a9e8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,3 @@ +Source code in this repository is variously licensed under the Business Source License 1.1 (BSL) and the Apache 2.0 license. +A copy of each license can be found in the licenses directory. +Source code in a given file is licensed under the BSL and the copyright belongs to RAW Labs S.A. unless otherwise noted at the beginning of the file. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..575d215 --- /dev/null +++ b/build.sbt @@ -0,0 +1,150 @@ +import sbt.Keys._ +import sbt._ + +import java.nio.file.Paths + +ThisBuild / credentials += Credentials( + "GitHub Package Registry", + "maven.pkg.github.com", + "raw-labs", + sys.env.getOrElse("GITHUB_TOKEN", "") +) + +val isRelease = sys.props.getOrElse("release", "false").toBoolean + +lazy val commonSettings = Seq( + homepage := Some(url("https://www.raw-labs.com/")), + organization := "com.raw-labs", + organizationName := "RAW Labs SA", + startYear := Some(2023), + organizationHomepage := Some(url("https://www.raw-labs.com/")), + developers := List(Developer("raw-labs", "RAW Labs", "engineering@raw-labs.com", url("https://github.com/raw-labs"))), + licenses := List( + "Business Source License 1.1" -> new URI( + "https://raw.githubusercontent.com/raw-labs/snapi/main/licenses/BSL.txt" + ).toURL + ), + headerSources / excludeFilter := HiddenFileFilter, + // Use cached resolution of dependencies + // http://www.scala-sbt.org/0.13/docs/Cached-Resolution.html + updateOptions := updateOptions.in(Global).value.withCachedResolution(true), + resolvers ++= Seq(Resolver.mavenLocal), + resolvers ++= Resolver.sonatypeOssRepos("snapshots"), + resolvers ++= Resolver.sonatypeOssRepos("releases") +) + +lazy val buildSettings = Seq( + scalaVersion := "2.12.18", + isSnapshot := !isRelease, + javacOptions ++= Seq( + "-source", + "21", + "-target", + "21" + ), + scalacOptions ++= Seq( + "-feature", + "-unchecked", + // When compiling in encrypted drives in Linux, the max size of a name is reduced to around 140 + // https://unix.stackexchange.com/a/32834 + "-Xmax-classfile-name", + "140", + "-deprecation", + "-Xlint:-stars-align,_", + "-Ywarn-dead-code", + "-Ywarn-macros:after", // Fix for false warning of unused implicit arguments in traits/interfaces. + "-Ypatmat-exhaust-depth", + "160" + ) +) + +lazy val compileSettings = Seq( + Compile / doc / sources := Seq.empty, + Compile / packageDoc / mappings := Seq(), + Compile / packageSrc / publishArtifact := true, + Compile / packageDoc / publishArtifact := false, + Compile / packageBin / packageOptions += Package.ManifestAttributes( + "Automatic-Module-Name" -> name.value.replace('-', '.') + ), + // Add all the classpath to the module path. + Compile / javacOptions ++= Seq( + "--module-path", + (Compile / dependencyClasspath).value.files.absString + ), + // The module-info.java requires the Scala classes to be compiled. + compileOrder := CompileOrder.ScalaThenJava +) + +lazy val testSettings = Seq( + // Exclude module-info.java, otherwise it will fail the compilation. + Test / doc / sources := { + (Compile / doc / sources).value.filterNot(_.getName.endsWith("module-info.java")) + }, + // Ensuring tests are run in a forked JVM for isolation. + Test / fork := true, + // Pass system properties starting with "raw." to the forked JVMs. + Test / javaOptions ++= { + import scala.collection.JavaConverters._ + val props = System.getProperties + props + .stringPropertyNames() + .asScala + .filter(_.startsWith("raw.")) + .map(key => s"-D$key=${props.getProperty(key)}") + .toSeq + }, + // Set up heap dump options for out-of-memory errors. + Test / javaOptions ++= Seq( + "-XX:+HeapDumpOnOutOfMemoryError", + s"-XX:HeapDumpPath=${Paths.get(sys.env.getOrElse("SBT_FORK_OUTPUT_DIR", "target/test-results")).resolve("heap-dumps")}" + ), + Test / publishArtifact := true, + Test / packageSrc / publishArtifact := true +) + +val isCI = sys.env.getOrElse("CI", "false").toBoolean + +lazy val publishSettings = Seq( + versionScheme := Some("early-semver"), + publish / skip := false, + publishMavenStyle := true, + publishTo := Some("GitHub raw-labs Apache Maven Packages" at "https://maven.pkg.github.com/raw-labs/utils-core"), + publishConfiguration := publishConfiguration.value.withOverwrite(isCI) +) + +lazy val strictBuildSettings = commonSettings ++ compileSettings ++ buildSettings ++ testSettings ++ Seq( + scalacOptions ++= Seq( + "-Xfatal-warnings" + ) +) + +lazy val root = (project in file(".")) + .settings( + name := "utils-core", + strictBuildSettings, + publishSettings, + libraryDependencies ++= Seq( + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5-rawlabs", + "com.typesafe" % "config" % "1.4.2", + "ch.qos.logback" % "logback-classic" % "1.4.12", + "com.google.guava" % "guava" % "32.1.3-jre", + "org.scala-lang.modules" %% "scala-java8-compat" % "1.0.2", // Required while we are on Scala 2.12. It's built into Scala 2.13. + "com.github.loki4j" % "loki-logback-appender" % "1.4.2", + "commons-io" % "commons-io" % "2.11.0", + "org.apache.commons" % "commons-text" % "1.11.0", + "org.slf4j" % "slf4j-api" % "2.0.5", + "org.slf4j" % "log4j-over-slf4j" % "2.0.5", + "org.slf4j" % "jcl-over-slf4j" % "2.0.5", + "org.slf4j" % "jul-to-slf4j" % "2.0.5", + "com.fasterxml.jackson.core" % "jackson-core" % "2.15.2", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2", + "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.15.2", + "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % "2.15.2", + "com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % "2.15.2", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.15.2", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-csv" % "2.15.2", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % "2.15.2", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.2-rawlabs", + "org.scalatest" %% "scalatest" % "3.2.16" % Test + ) + ) \ No newline at end of file diff --git a/licenses/APL.txt b/licenses/APL.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/licenses/APL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/BSL.txt b/licenses/BSL.txt new file mode 100644 index 0000000..99588db --- /dev/null +++ b/licenses/BSL.txt @@ -0,0 +1,103 @@ +Business Source License 1.1 + +Parameters + +Licensor: RAW Labs S.A. +Licensed Work: RAW + The Licensed Work is (c) 2015 RAW Labs S.A. +Additional Use Grant: You may make use of the Licensed Work, provided that + you may not use the Licensed Work for a Cloud + Service. + + A “Cloud Service” is a commercial offering that + allows third parties (other than your employees and + contractors) to access the functionality of the + Licensed Work to create programs or APIs whose source are + controlled by such third parties. + +Change Date: 2027-01-01 + +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please visit: https://cockroachlabs.com/ + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/project/CopyrightHeader.scala b/project/CopyrightHeader.scala new file mode 100644 index 0000000..e23e16a --- /dev/null +++ b/project/CopyrightHeader.scala @@ -0,0 +1,65 @@ +import de.heikoseeberger.sbtheader.HeaderPlugin.autoImport._ +import de.heikoseeberger.sbtheader.{CommentCreator, HeaderPlugin} +import sbt.Keys._ +import sbt._ + +object CopyrightHeader extends AutoPlugin { + + override def requires: Plugins = HeaderPlugin + + override def trigger: PluginTrigger = allRequirements + + protected def headerMappingSettings: Seq[Def.Setting[_]] = Seq(Compile, Test).flatMap { config => + inConfig(config)( + Seq( + headerLicense := Some(HeaderLicense.Custom(header)), + headerMappings := headerMappings.value ++ Map( + HeaderFileType.scala -> cStyleComment, + HeaderFileType.java -> cStyleComment + ) + ) + ) + } + + override def projectSettings: Seq[Def.Setting[_]] = Def.settings(headerMappingSettings, additional) + + def additional: Seq[Def.Setting[_]] = Def.settings( + Compile / compile := { + (Compile / headerCreate).value + (Compile / compile).value + }, + Test / compile := { + (Test / headerCreate).value + (Test / compile).value + } + ) + + def header: String = { + val currentYear = "2024" + s"""|/* + | * Copyright $currentYear RAW Labs S.A. + | * + | * Use of this software is governed by the Business Source License + | * included in the file licenses/BSL.txt. + | * + | * As of the Change Date specified in that file, in accordance with + | * the Business Source License, use of this software will be governed + | * by the Apache License, Version 2.0, included in the file + | * licenses/APL.txt. + | */""".stripMargin + } + + val cStyleComment = HeaderCommentStyle.cStyleBlockComment.copy(commentCreator = new CommentCreator() { + val CopyrightPattern = "Copyright (\\d{4}) RAW Labs S.A.".r + + override def apply(text: String, existingText: Option[String]): String = { + existingText match { + case Some(existingHeader) if CopyrightPattern.findFirstIn(existingHeader).isDefined => + // matches the pattern with any year, return it unchanged + existingHeader.trim + case _ => header + } + } + }) + +} diff --git a/project/build.properties b/project/build.properties new file mode 100755 index 0000000..2743082 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.6 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100755 index 0000000..c5b285c --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,15 @@ +resolvers += Classpaths.sbtPluginReleases + +autoCompilerPlugins := true + +addDependencyTreePlugin + +libraryDependencies += "commons-io" % "commons-io" % "2.11.0" + +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.0") + +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") + +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.12.0") + +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1") diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java new file mode 100644 index 0000000..75e2930 --- /dev/null +++ b/src/main/java/module-info.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +module raw.utils.core { + requires scala.library; + requires com.fasterxml.jackson.core; + requires com.fasterxml.jackson.databind; + requires com.fasterxml.jackson.dataformat.csv; + requires com.fasterxml.jackson.scala; + requires org.apache.commons.io; + requires org.apache.commons.text; + requires typesafe.config; + requires typesafe.scalalogging; + requires org.slf4j; + requires ch.qos.logback.classic; + requires com.google.common; + requires jul.to.slf4j; + + exports com.rawlabs.utils.core; + + opens com.rawlabs.utils.core to + com.fasterxml.jackson.databind; +} diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf new file mode 100644 index 0000000..f70cd01 --- /dev/null +++ b/src/main/resources/reference.conf @@ -0,0 +1,6 @@ +raw { + # Generic flag to enable/disable training wheels. + # Training wheels are a set of features that help the user to understand the system. + # They are not meant to be used in production as they can slow down the system significantly. + training-wheels = false +} diff --git a/src/main/scala/com/rawlabs/utils/core/RawException.scala b/src/main/scala/com/rawlabs/utils/core/RawException.scala new file mode 100644 index 0000000..d9010a9 --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +/** + * Top-level Exception. + * Message contains information that WILL BE shared with the end-user, so ensure it does not leak sensitive information. + */ +class RawException(message: String, cause: Throwable = null) extends RuntimeException(message, cause) diff --git a/src/main/scala/com/rawlabs/utils/core/RawMBeansManager.scala b/src/main/scala/com/rawlabs/utils/core/RawMBeansManager.scala new file mode 100644 index 0000000..ae9c0df --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawMBeansManager.scala @@ -0,0 +1,100 @@ +/* + * Copyright 2024 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.scalalogging.StrictLogging + +import java.lang.management.ManagementFactory +import javax.management.ObjectName +import scala.util.control.NonFatal + +/** + * Base Trait for MBeans + */ +trait RawMBean { + + /** + * Get the name of the MBean + * + * @return the name of the MBean + */ + def getMBeanName: String +} + +object RawMBeansManager extends StrictLogging { + + private val mbs = ManagementFactory.getPlatformMBeanServer + private var mbeanMap = Map.empty[String, RawMBean] + private val mbeanLock = new Object + + /** + * Register an MBean + * + * @param mbean the MBean to register + */ + def registerMBean(mbean: RawMBean): Unit = { + require(mbean != null, "MBean cannot be null") + require(mbean.getMBeanName != null, "MBean name cannot be null") + mbeanLock.synchronized { + if (mbs.isRegistered(new ObjectName(mbean.getMBeanName))) { + logger.warn(s"MBean ${mbean.getMBeanName} already registered") + } else { + val objectName = new ObjectName(mbean.getMBeanName) + mbs.registerMBean(mbean, objectName) + mbeanMap += (mbean.getMBeanName -> mbean) + } + } + } + + /** + * Unregister an MBean + * + * @param name the name of the MBean to unregister + */ + def unregisterMBean(name: String): Unit = { + require(name != null, "MBean name cannot be null") + mbeanLock.synchronized { + val objectName = new ObjectName(name) + if (mbs.isRegistered(objectName)) { + mbs.unregisterMBean(objectName) + mbeanMap -= name + } else { + logger.warn(s"MBean $name is not registered") + } + } + } + + /** + * Get a copy of an MBean + * + * @param name the name of the MBean to get + * @tparam T the type of the MBean + * @return an option containing the MBean if it exists, None otherwise + */ + def getMBeanCopy[T](name: String): Option[T] = { + require(name != null, "MBean name cannot be null") + mbeanLock.synchronized { + try { + val objectName = new ObjectName(name) + if (mbs.isRegistered(objectName)) { + Some(mbeanMap(name).asInstanceOf[T]) + } else { + None + } + } catch { + case NonFatal(_) => None + } + } + } + +} diff --git a/src/main/scala/com/rawlabs/utils/core/RawService.scala b/src/main/scala/com/rawlabs/utils/core/RawService.scala new file mode 100644 index 0000000..6a3b584 --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawService.scala @@ -0,0 +1,95 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.scalalogging.StrictLogging + +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicBoolean +import scala.util.control.NonFatal + +object RawService { + + private[rawlabs] val services = new LinkedBlockingQueue[RawService] + + /** Used by main's to stop all created services. */ + def stopAll(): Unit = { + services.forEach(s => s.stop()) + } + + /** Used by test framework to assert all services were stopped. */ + private[rawlabs] def isStopped(): Boolean = { + services.size() == 0 + } + +} + +/** + * RAW Service. + * + * Used for services (aka. components/modules) that may require centralized stopping. + */ +trait RawService extends StrictLogging { + + import RawService._ + protected val stopped = new AtomicBoolean(false) + + logger.debug(s"Adding service: $this") + services.add(this) + + /** + * Stop the service. + * After stop() is called, any future calls, including to stop(), will result in undefined behaviour. + */ + final def stop(): Unit = { + if (stopped.compareAndSet(false, true)) { + try { + doStop() + } catch { + case NonFatal(t) => + // Do nothing. + logger.warn(s"Stopping service $this failed with NonFatal.", t) + throw t + } finally { + val removed = services.remove(this) + if (removed) { + logger.debug(s"Stopping service: $this") + } else { + logger.warn(s"Service was not found on active service list: $this") + } + } + } else { + logger.debug( + s"Service already stopped: $this. Caller:\n${Thread.currentThread().getStackTrace.take(10).mkString("\t\n")}" + ) + } + } + + /** + * Implementors CANNOT THROW AN EXCEPTION inside doStop()! Refer to implementation of stop(). + * Otherwise, we may eat the original exception and throw a new exception inside doStop() caused by accessing + * partially initialized classes. + * + * Purposely not implemented so that children do not need to artificial call super.doStop() + */ + protected def doStop(): Unit + +} + +/** + * Exception thrown by a service. + * + * @param message the exception message + * @param cause the exception cause + */ +class RawServiceException(message: String, cause: Throwable = null) extends RawException(message, cause) diff --git a/src/main/scala/com/rawlabs/utils/core/RawSettings.scala b/src/main/scala/com/rawlabs/utils/core/RawSettings.scala new file mode 100644 index 0000000..31d6a5b --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawSettings.scala @@ -0,0 +1,387 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.config._ +import com.typesafe.scalalogging.StrictLogging + +import java.time.Duration +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class SettingsException(message: String, cause: Throwable = null) extends RawException(message, cause) + +object RawSettings { + + private val alreadyLoggedLock = new Object + private val alreadyLogged = new mutable.HashSet[(String, Any)] + +} + +class RawSettings( + protected val lowPriorityConfig: Config = ConfigFactory.load(), + protected val highPriorityConfig: Config = ConfigFactory.empty() +) extends StrictLogging { + + import RawSettings._ + + val config: Config = highPriorityConfig.withFallback(lowPriorityConfig) + + private def logOneTime(key: String, value: Any, propertyType: String): Unit = { + alreadyLoggedLock.synchronized { + if (alreadyLogged.add((key, value))) { + logger.info(s"Using $key: $value ($propertyType)") + } + } + } + + private def withLogConfigException[T](propertyName: String, f: () => T): T = { + try { + f() + } catch { + case ex: ConfigException => throw new SettingsException(s"error loading property: $propertyName", ex) + } + } + + def this(renderAsString: String) = { + this( + try { + ConfigFactory.parseString(renderAsString) + } catch { + case ex: ConfigException => throw new SettingsException("error loading settings", ex) + } + ) + } + + def renderAsString: String = { + try { + config.root().render(ConfigRenderOptions.concise()) + } catch { + case ex: ConfigException => throw new SettingsException(s"error writing configuration: ${ex.getMessage}", ex) + } + } + + private def convertConfigToFriendlyConfSyntax(config: Config): Set[(String, String)] = { + config + .entrySet() + .asScala + .map { t => + val key = t.getKey + config.getValue(key).valueType() match { + case ConfigValueType.NUMBER => (key -> config.getNumber(key).toString) + case ConfigValueType.BOOLEAN => (key -> config.getBoolean(key).toString) + case ConfigValueType.STRING => (key -> config.getString(key)) + case ConfigValueType.NULL => (key -> "null") + case ConfigValueType.OBJECT => + throw new IllegalStateException(s"found unsupported object type in key $key for spark configuration") + case ConfigValueType.LIST => (key -> config.getStringList(key).toString) + } + } + .to + } + + override def toString: String = { + val settings = convertConfigToFriendlyConfSyntax(config) + settings + .map(entry => s"${entry._1} -> ${entry._2}") + .mkString("; ") + } + + def withModuleConfig(moduleConfig: Config): RawSettings = { + val config: Config = ConfigFactory + .defaultOverrides() + .withFallback(ConfigFactory.defaultApplication()) + .withFallback(moduleConfig) + .withFallback(ConfigFactory.defaultReference()) + .resolve() + new RawSettings(config, highPriorityConfig) + } + + def withFallback(settings: RawSettings): RawSettings = { + new RawSettings( + lowPriorityConfig.withFallback(settings.lowPriorityConfig), + highPriorityConfig.withFallback(settings.highPriorityConfig) + ) + } + + def cloneWith(settings: Map[String, Any]): RawSettings = { + new RawSettings(lowPriorityConfig, ConfigFactory.parseMap(settings.asJava).withFallback(highPriorityConfig)) + } + + def cloneWith(settings: String): RawSettings = { + new RawSettings(lowPriorityConfig, ConfigFactory.parseString(settings).withFallback(highPriorityConfig)) + } + + def cloneWith(settings: (String, Any)*): RawSettings = { + cloneWith(settings.toMap) + } + + def getStringList(property: String): Seq[String] = { + withLogConfigException( + property, + () => { + val v = config.getStringList(property) + logOneTime(property, v, "stringList") + v.asScala + } + ) + } + + def getStringListOpt(property: String): Option[Seq[String]] = { + withLogConfigException( + property, + () => + try { + val v = config.getStringList(property) + logOneTime(property, v, "stringList") + Some(v.asScala) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, "stringList") + None + } + ) + } + + def getString(property: String, logValue: Boolean = true): String = { + withLogConfigException( + property, + () => { + val v = config.getString(property) + if (logValue) { + logOneTime(property, v, "string") + } else { + logOneTime(property, "*****", "string") + } + v + } + ) + } + + def getStringOpt(property: String, logValue: Boolean = true): Option[String] = { + withLogConfigException( + property, + () => { + try { + val v = config.getString(property) + if (logValue) { + logOneTime(property, v, "string") + } else { + logOneTime(property, "*****", "string") + } + Some(v) + } catch { + case _: ConfigException.Missing => { + logOneTime(property, None, "string") + None + } + } + } + ) + } + + def getBooleanOpt(property: String, logValue: Boolean = true): Option[Boolean] = { + withLogConfigException( + property, + () => { + try { + val v = config.getBoolean(property) + if (logValue) { + logOneTime(property, v, "boolean") + } else { + logOneTime(property, "*****", "boolean") + } + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, "boolean") + None + } + } + ) + } + + def getBytes(property: String): Long = { + withLogConfigException( + property, + () => { + val v = config.getBytes(property) + logOneTime(property, v, "bytes") + v + } + ) + } + + def getBytesOpt(property: String, logValue: Boolean = true): Option[Long] = { + withLogConfigException( + property, + () => { + try { + val v = config.getBytes(property) + if (logValue) { + logOneTime(property, v, "bytes") + } else { + logOneTime(property, "*****", "bytes") + } + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, "bytes") + None + } + } + ) + } + + def getInt(property: String): Int = { + withLogConfigException( + property, + () => { + val v = config.getInt(property) + logOneTime(property, v, "int") + v + } + ) + } + + def getIntOpt(property: String, logValue: Boolean = true): Option[Int] = { + withLogConfigException( + property, + () => { + try { + val v = config.getInt(property) + if (logValue) { + logOneTime(property, v, "int") + } else { + logOneTime(property, "*****", "int") + } + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, "int") + None + } + } + ) + } + + def getMemorySize(property: String): Long = { + withLogConfigException( + property, + () => { + val v = config.getMemorySize(property).toBytes + logOneTime(property, v, "long") + v + } + ) + } + + def getBoolean(property: String): Boolean = { + withLogConfigException( + property, + () => { + val v = config.getBoolean(property) + logOneTime(property, v, "boolean") + v + } + ) + } + + def getBooleanOpt(property: String): Option[Boolean] = { + withLogConfigException( + property, + () => { + try { + val v = config.getBoolean(property) + logOneTime(property, v, s"boolean") + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, s"boolean") + None + } + } + ) + } + + def getDuration(property: String): Duration = { + withLogConfigException( + property, + () => { + val v = config.getDuration(property) + logOneTime(property, v, "duration") + v + } + ) + } + + def getDuration(property: String, timeUnit: TimeUnit): Long = { + withLogConfigException( + property, + () => { + val v = config.getDuration(property, timeUnit) + logOneTime(property, v, s"duration with timeUnit $timeUnit") + v + } + ) + } + + def getDurationOpt(property: String, timeUnit: TimeUnit): Option[Long] = { + withLogConfigException( + property, + () => { + try { + val v = config.getDuration(property, timeUnit) + logOneTime(property, v, s"duration with timeUnit $timeUnit") + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, s"duration with timeUnit $timeUnit") + None + } + } + ) + } + + def getDurationOpt(property: String): Option[Duration] = { + withLogConfigException( + property, + () => { + try { + val v = config.getDuration(property) + logOneTime(property, v, "duration") + Some(v) + } catch { + case _: ConfigException.Missing => + logOneTime(property, None, "duration") + None + } + } + ) + } + + def getDouble(property: String): Double = { + withLogConfigException( + property, + () => { + val v = config.getDouble(property) + logOneTime(property, v, "double") + v + } + ) + } + + val onTrainingWheels: Boolean = getBooleanOpt("raw.training-wheels").getOrElse(false) + +} diff --git a/src/main/scala/com/rawlabs/utils/core/RawUid.scala b/src/main/scala/com/rawlabs/utils/core/RawUid.scala new file mode 100644 index 0000000..438ccc4 --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawUid.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +final case class RawUid(uid: String) { + override def toString: String = uid +} diff --git a/src/main/scala/com/rawlabs/utils/core/RawUtils.scala b/src/main/scala/com/rawlabs/utils/core/RawUtils.scala new file mode 100644 index 0000000..157f2eb --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawUtils.scala @@ -0,0 +1,339 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.google.common.io.Resources +import com.google.common.util.concurrent.ThreadFactoryBuilder +import com.typesafe.scalalogging.StrictLogging +import org.apache.commons.io.FileUtils + +import java.io.{ByteArrayOutputStream, InputStream} +import java.math.{MathContext, RoundingMode} +import java.net.URL +import java.nio.charset.{Charset, StandardCharsets} +import java.nio.file.{FileSystemNotFoundException, Files, Path, Paths, StandardCopyOption} +import java.util.Locale +import java.util.concurrent.{ExecutorService, SynchronousQueue, ThreadFactory, ThreadPoolExecutor, TimeUnit} +import java.util.zip.ZipFile +import scala.util.control.NonFatal +import scala.collection.JavaConverters._ +import org.apache.commons.text.StringEscapeUtils + +/** + * "Random" collection of utility methods. + * + * NOTE that this classes uses the `StrictLogging` trait from the `com.typesafe.scalalogging` package. + * TODO (msb): Remove StrictLogging and make logger a parameter where needed. + */ +object RawUtils extends StrictLogging { + + /** + * Convert a user string back to its original "intended" representation. + * e.g. if the user types "\t" we get the a single '\t' char out instead of the two byte string "\t". + */ + def escape(s: String): String = StringEscapeUtils.unescapeJava(s) + + /** Does the opposite of the method `escape`. */ + def descape(s: String): String = StringEscapeUtils.escapeJava(s) + + def readEntireFile(path: Path, charset: Charset = StandardCharsets.UTF_8): String = { + new String(Files.readAllBytes(path), charset) + } + + /** + * Creates a cached thread pool that scales the number of threads from `minSize` to `maxSize`. This avoids keeping around + * idle threads when they are not needed. The factory methods in the `Executors` class do not allow specifying the min and + * max number of threads, they default to 0 and Integer.MAX_VALUE. + * + * @param minSize + * @param maxSize + * @param name + * @return + */ + def newBoundedCachedThreadPool(minSize: Int, maxSize: Int, name: String): ExecutorService = { + new ThreadPoolExecutor( + minSize, + maxSize, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue[Runnable], + newThreadFactory(name) + ) + } + + def newThreadFactory(name: String, daemon: Boolean = true): ThreadFactory = { + new ThreadFactoryBuilder() + .setNameFormat(s"$name-%d") + .setUncaughtExceptionHandler((t: Thread, e: Throwable) => { + logger.warn(s"Uncaught exception on thread: ${t.getName}", e) + }) + .setDaemon(daemon) + .build() + } + + def readInputStreamAsString(inputStream: InputStream, charset: Charset = StandardCharsets.UTF_8): String = { + val result = new ByteArrayOutputStream() + val buffer = new Array[Byte](1024) + var length: Int = inputStream.read(buffer) + while (length != -1) { + result.write(buffer, 0, length) + length = inputStream.read(buffer) + } + result.toString(charset.name) + } + + def endsWithIgnoreCase(str: String, ch: Char): Boolean = { + val last = str.charAt(str.length - 1) + Character.toLowerCase(last) == Character.toLowerCase(ch) + } + + lazy val isWindows: Boolean = { + System.getProperty("os.name").contains("Windows") + } + + lazy val isMacOS: Boolean = { + System.getProperty("os.name").toLowerCase().contains("mac os x") + } + + def withSuppressNonFatalException(f: => Unit, silent: Boolean = false): Unit = { + if (Thread.interrupted()) { + throw new InterruptedException() + } + try { + f + } catch { + case NonFatal(t) => if (!silent) { + logger.warn("Suppressing uncaught exception: ", t) + } + } + } + + def withSuppressNonFatalExceptionUninterruptible(f: => Unit, silent: Boolean = false): Unit = { + try { + f + } catch { + case NonFatal(t) => if (!silent) { + logger.warn("Suppressing uncaught exception: ", t) + } + } + } + + def deleteTestPath(path: Path): Unit = { + if (Files.isDirectory(path)) { + deleteTestDirectory(path) + } else { + deleteTestFile(path) + } + } + + def deleteTestDirectory(directory: Path): Unit = { + withSuppressNonFatalException { + FileUtils.deleteDirectory(directory.toFile) + } + } + + def deleteTestFile(file: Path): Unit = { + withSuppressNonFatalException { + val deleted = Files.deleteIfExists(file) + if (!deleted) { + logger.warn(s"Could not delete test file: $file") + } + } + } + + def stackTraceToString(t: Throwable, maxStack: Int = Integer.MAX_VALUE): String = { + val sb = new StringBuilder + sb.append(t.toString + "\n") + sb.append( + t.getStackTrace + .take(maxStack) + .mkString("\tat ", "\n\tat ", "") + ) + if (maxStack < t.getStackTrace.length) { + sb.append("\n\t...") + } + if (t.getCause != null) { + sb.toString + "\nCaused by: " + stackTraceToString(t.getCause, maxStack) + } else { + sb.toString() + } + } + + def getTemporaryPath: Path = Paths.get(System.getProperty("java.io.tmpdir")) + + def getTemporaryPath(dirname: String): Path = Paths.get(System.getProperty("java.io.tmpdir"), dirname) + + private val tempDirLock = new Object + private var tempDir: Path = _ + + def getResource(resource: String): Path = { + val maybeResourcePath = { + try { + val resourcePath = Paths.get(Resources.getResource(resource).toURI) + if (Files.exists(resourcePath)) { + Some(resourcePath) + } else { + None + } + } catch { + case _: FileSystemNotFoundException => None + } + } + maybeResourcePath match { + case Some(resourcePath) => resourcePath + case None => + // (msb) The following is a hack to load test data. + // We have test files in src/test/resources but these are in a JAR test dependency. + // In that case, we cannot read them as a file system: JARs because ZIPs and we cannot just read or list their contents. + // Therefore, if we are trying to access data/ files, we inspect the classpath for 'raw-sources' JARs. + // We then open those up, and create a temporary directory where their data is placed. + // This allows us to read it from that temp directory as if they were in a regular file system. + if (resource.startsWith("data/")) { + val classpath = System.getProperty("java.class.path") + val classpathEntries = classpath.split(System.getProperty("path.separator")) + val jarFiles = classpathEntries.filter(p => p.contains("snapi-") && p.endsWith("-tests.jar")) + if (jarFiles.isEmpty) { + // We are not running in JAR mode, so just return the resource directly. + Paths.get(Resources.getResource(resource).toURI) + } else { + tempDirLock.synchronized { + // We are running in JAR mode, so read all data/ files in the JARs and copy them over to temporary directories. + if (tempDir == null) { + tempDir = Files.createTempDirectory("resource") + jarFiles.foreach { jarFile => + val jar = new ZipFile(jarFile) + try { + jar.entries.asScala.foreach { entry => + if (entry.getName.startsWith("data/") && !entry.isDirectory) { + val outputPath = tempDir.resolve(entry.getName) + Files.createDirectories(outputPath.getParent) + val in = jar.getInputStream(entry) + try { + Files.copy(in, outputPath, StandardCopyOption.REPLACE_EXISTING) + } finally { + in.close() + } + } + } + } finally { + jar.close() + } + } + } + } + tempDir.resolve(resource) + } + } else { + // If not running from a JAR, can access it directly. + Paths.get(Resources.getResource(resource).toURI) + } + } + } + + /** + * The deleteOnExit() will leave the file around until the JVM exits. This will lead to a large + * number of tmp files left behind when running tests from InteliJ or in the CI server, which + * run the tests in the host JVM. There's also the risk of registering a potentially unlimited + * number of files with the delete hook of the JVM. + */ + def saveToTemporaryFileNoDeleteOnExit( + contents: String, + prefix: String, + postfix: String, + charset: Charset = StandardCharsets.UTF_8 + ): Path = { + val p: Path = Files.createTempFile(prefix, postfix) + Files.write(p, contents.getBytes(charset)) + p + } + + /** + * Creates a temporary file with the given contents, runs the lamdba given in the second argument + * list, then delete the temporary file. + */ + def withSavedAsTemporaryFile[T]( + contents: String, + prefix: String, + postfix: String, + charset: Charset = StandardCharsets.UTF_8 + )(f: (Path) => T): T = { + val path = saveToTemporaryFileNoDeleteOnExit(contents, prefix, postfix, charset) + try { + f(path) + } finally { + Files.delete(path) + } + } + + def readResource(resource: String): String = { + val release: URL = Resources.getResource(resource) + Resources.toString(release, StandardCharsets.UTF_8) + } + + def escapeLanguage(code: String): String = { + val tquote = "\"\"\"" + s"""$tquote ${descape(code)} $tquote""".stripMargin + } + + def bytesToString(size: Long): String = bytesToString(BigInt(size)) + + def bytesToString(size: BigInt): String = { + val EiB = 1L << 60 + val PiB = 1L << 50 + val TiB = 1L << 40 + val GiB = 1L << 30 + val MiB = 1L << 20 + val KiB = 1L << 10 + + if (size >= BigInt(1L << 11) * EiB) { + // The number is too large, show it in scientific notation. + BigDecimal(size, new MathContext(3, RoundingMode.HALF_UP)).toString() + " B" + } else { + val (value, unit) = { + if (size >= 2 * EiB) { + (BigDecimal(size) / EiB, "EiB") + } else if (size >= 2 * PiB) { + (BigDecimal(size) / PiB, "PiB") + } else if (size >= 2 * TiB) { + (BigDecimal(size) / TiB, "TiB") + } else if (size >= 2 * GiB) { + (BigDecimal(size) / GiB, "GiB") + } else if (size >= 2 * MiB) { + (BigDecimal(size) / MiB, "MiB") + } else if (size >= 2 * KiB) { + (BigDecimal(size) / KiB, "KiB") + } else { + (BigDecimal(size), "B") + } + } + "%.1f %s".formatLocal(Locale.US, value, unit) + } + } + + def trimNumbers(v: String): String = { + val tv = v.trim + val isNeg = tv.head == '-' + if (isNeg) { + val cv = tv.drop(1).dropWhile(_ == '0') + if (cv.isEmpty) "-0" else "-" + cv + } else { + val cv = tv.dropWhile(_ == '0') + if (cv.isEmpty) "0" else cv + } + } + + def getFileNameWithoutExtension(path: Path): String = { + com.google.common.io.Files.getNameWithoutExtension(path.getFileName.toString) + } + +} diff --git a/src/main/scala/com/rawlabs/utils/core/RawVerboseRunnable.scala b/src/main/scala/com/rawlabs/utils/core/RawVerboseRunnable.scala new file mode 100644 index 0000000..79451c6 --- /dev/null +++ b/src/main/scala/com/rawlabs/utils/core/RawVerboseRunnable.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.scalalogging.StrictLogging + +/** + * Wraps a Runnable to log any uncaught exceptions. + * + * This is useful for tasks that run on the background without any supervision from a parent thread, so that if they fail + * with an exception, there is no one to catch and handle it. This is opposed to cases where some parent thread will handle + * the result of the task, either by calling Future.get or by adding an error handling stage when using completable + * futures. + * + * Inspired by: + * https://github.com/jcabi/jcabi-log/blob/master/src/main/java/com/jcabi/log/VerboseRunnable.java + * + * @param delegate The runnable to wrap. + * @param propagate Whether to rethrow the exception. + */ +class RawVerboseRunnable(delegate: Runnable, propagate: Boolean = false) extends Runnable with StrictLogging { + override def run(): Unit = { + try { + delegate.run() + } catch { + case t: Throwable => + logger.warn("Uncaught error executing runnable", t) + if (propagate) { + throw t + } + } + } +} diff --git a/src/test/scala/com/rawlabs/utils/core/RawMultiplyingTestSuite.scala b/src/test/scala/com/rawlabs/utils/core/RawMultiplyingTestSuite.scala new file mode 100644 index 0000000..3b388b9 --- /dev/null +++ b/src/test/scala/com/rawlabs/utils/core/RawMultiplyingTestSuite.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import org.scalatest.{Args, CompositeStatus, Status} + +trait RawMultiplyingTestSuite extends RawTestSuite { + this: SettingsTestContext => + + private var settingsToUse = Seq.empty[Map[String, Any]] + + protected var currentFormat: String = _ + + def multiplyingSettings(settings: Seq[Map[String, String]]): Unit = { + settingsToUse = settings + } + + override def run(testName: Option[String], args: Args): Status = { + if (settingsToUse.isEmpty) { + // Run tests normally. + super.run(testName, args) + } else { + assert(testName.isEmpty) + val statuses = for (s <- settingsToUse) yield { + val originalProperties = properties.clone() + try { + s.foreach { case (k, v) => properties.put(k, v) } + super.run(testName, args) + } finally { + properties = originalProperties + } + } + new CompositeStatus(statuses.to) + } + } + +} diff --git a/src/test/scala/com/rawlabs/utils/core/RawTestSuite.scala b/src/test/scala/com/rawlabs/utils/core/RawTestSuite.scala new file mode 100644 index 0000000..8602194 --- /dev/null +++ b/src/test/scala/com/rawlabs/utils/core/RawTestSuite.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.scalalogging.StrictLogging +import org.scalatest.funsuite.FixtureAnyFunSuite +import org.scalatest.{BeforeAndAfterAll, Outcome} + +import java.lang.management.ManagementFactory +import java.util.concurrent.atomic.AtomicBoolean +import scala.collection.JavaConverters._ + +case class TestData(q: String) + +object RawTestSuite extends StrictLogging { + private val called = new AtomicBoolean(false) + + def printJvmInfo(): Unit = { + if (called.compareAndSet(false, true)) { + val runtimeMxBean = ManagementFactory.getRuntimeMXBean + val arguments = runtimeMxBean.getInputArguments + logger.debug(s"Runtime arguments: ${arguments.asScala.mkString("\n ", "\n ", "")}") + } + } +} + +trait RawTestSuite extends FixtureAnyFunSuite with BeforeAndAfterAll with StrictLogging { + RawTestSuite.printJvmInfo() + + type FixtureParam = TestData + + protected def withFixture(test: OneArgTest): Outcome = { + test(TestData(test.name)) + } + + override def beforeAll(): Unit = { + // If a previous test suite crashed, clean its leftovers anyway. + RawService.stopAll() + super.beforeAll() + } + + override def afterAll(): Unit = { + // Always ensure that all services have been stopped correctly. + // If not, this means the code under test is not cleaning up properly. + logger.info("Checking if all services have stopped") + var attempts = 10 + while (!RawService.isStopped() && attempts > 0) { + attempts -= 1 + logger.debug(s"Waiting for services to terminate gracefully. Attempts left: $attempts") + Thread.sleep(1000) + } + assert(RawService.isStopped(), s"Not all services stopped properly. Still running: ${RawService.services}") + } +} diff --git a/src/test/scala/com/rawlabs/utils/core/SettingsTestContext.scala b/src/test/scala/com/rawlabs/utils/core/SettingsTestContext.scala new file mode 100644 index 0000000..f083d29 --- /dev/null +++ b/src/test/scala/com/rawlabs/utils/core/SettingsTestContext.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +import com.typesafe.config.ConfigFactory + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +trait SettingsTestContext { + protected var properties: mutable.Map[String, Any] = mutable.Map[String, Any]() + + implicit def settings: RawSettings = new RawSettings( + ConfigFactory.load(), + ConfigFactory.parseMap(properties.asJava) + ) + + def property(key: String, value: String): Unit = properties.put(key, value) + + def property(key: String, value: List[String]): Unit = properties.put(key, value.asJava) + + def property(key: String): Option[Any] = properties.get(key) +} diff --git a/src/test/scala/com/rawlabs/utils/core/TrainingWheelsContext.scala b/src/test/scala/com/rawlabs/utils/core/TrainingWheelsContext.scala new file mode 100644 index 0000000..af02975 --- /dev/null +++ b/src/test/scala/com/rawlabs/utils/core/TrainingWheelsContext.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2023 RAW Labs S.A. + * + * Use of this software is governed by the Business Source License + * included in the file licenses/BSL.txt. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0, included in the file + * licenses/APL.txt. + */ + +package com.rawlabs.utils.core + +trait TrainingWheelsContext { + this: RawTestSuite with SettingsTestContext => + + property("raw.training-wheels", "true") +}