diff --git a/README.md b/README.md index dad376c..c77ae32 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Scala 2.11 compatibility. ### Cross versions The project is using cross version to manage 2.11, 2.12 and 2.13 using the same code base, so remember to use '+' to -trigger all versions versions. +trigger all versions. So remember, as example, for testing: ```shell script diff --git a/build.sbt b/build.sbt index 3136000..31e8aa2 100644 --- a/build.sbt +++ b/build.sbt @@ -26,74 +26,15 @@ import sbt.Keys._ import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ import sbtrelease.ReleasePlugin.autoImport._ - -def isPatch211Enable(): Boolean = sys.env.getOrElse("PATCH_211", "false").toBoolean +import Dependencies._ +import CommonSettings._ // Dependencies -lazy val scalatestVersion = "3.2.0" -lazy val scalacheckVersion = "1.14.3" -lazy val commonIOVersion = "2.5" -lazy val logbackVersion = "1.1.7" -lazy val scoptVersion = "3.7.1" -lazy val akkaVersion = "2.5.31" -lazy val spark3Version = "3.1.1" -lazy val spark2Version = "2.4.7" lazy val sparkDefaultVersion = spark3Version -lazy val scala213 = "2.13.5" -lazy val scala212 = "2.12.13" -lazy val scala211 = "2.11.12" -lazy val scalaVersions = if (isPatch211Enable()) Seq(scala211) else Seq(scala213, scala212) -lazy val sparkScalaVersions = if (isPatch211Enable()) Seq(scala211) else Seq(scala212) - -lazy val commonSettings = Seq( - crossScalaVersions := scalaVersions, - organization := "com.acervera.osm4scala", - organizationHomepage := Some(url("https://www.acervera.com")), - licenses += ("MIT", url("https://opensource.org/licenses/MIT")), - ThisBuild / homepage := Some( - url(s"https://simplexspatial.github.io/osm4scala/") - ), - ThisBuild / scmInfo := Some( - ScmInfo( - url("https://github.com/simplexspatial/osm4scala"), - "scm:git:git://github.com/simplexspatial/osm4scala.git", - "scm:git:ssh://github.com:simplexspatial/osm4scala.git" - ) - ), - ThisBuild / developers := List( - Developer( - "angelcervera", - "Angel Cervera Claudio", - "angelcervera@silyan.com", - url("https://www.acervera.com") - ) - ), - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % scalatestVersion % Test, - "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test, - "commons-io" % "commons-io" % commonIOVersion % Test - ), - assembly / test := {}, - scalacOptions ++= Seq( - "-target:jvm-1.8", - "-encoding", - "utf8", - "-deprecation", - "-unchecked", - "-Xlint" - ), - javacOptions ++= Seq( - "-Xlint:all", - "-source", - "1.8", - "-target", - "1.8", - "-parameters" - ), - usePgpKeyHex("A047A2C5A9AFE4850537A00DFC14CE4C2E7B7CBB"), - publishTo := sonatypePublishToBundle.value -) +lazy val scala213 = "2.13.10" +lazy val scala212 = "2.12.17" +lazy val scalaAllVersions = Seq(scala212, scala213) lazy val disablingPublishingSettings = Seq(publish / skip := true, publishArtifact := false) @@ -115,16 +56,14 @@ def generateSparkFatShadedModule(sparkVersion: String, sparkPrj: Project): Proje Project( id = s"osm4scala-spark${sparkVersion.head}-shaded", base = file(s"target/osm4scala-spark${sparkVersion.head}-shaded") - ) - .disablePlugins(AssemblyPlugin) + ).disablePlugins(AssemblyPlugin) .settings( commonSettings, - crossScalaVersions := sparkScalaVersions, enablingPublishingSettings, disablingCoverage, name := s"osm4scala-spark${sparkVersion.head}-shaded", - description := "Spark 2 connector for OpenStreetMap Pbf parser as shaded fat jar.", - Compile / packageBin := (sparkPrj / Compile/ assembly).value + description := s"Spark ${sparkVersion.head} connector for OpenStreetMap Pbf parser as shaded fat jar.", + Compile / packageBin := (sparkPrj / Compile / assembly).value ) def generateSparkModule(sparkVersion: String): Project = { @@ -135,26 +74,26 @@ def generateSparkModule(sparkVersion: String): Project = { s"target/spark${sparkVersion.head}" } - def pathFromModule(relativePath: String): String = if (sparkDefaultVersion == sparkVersion) { - relativePath - } else { - s"../../spark/$relativePath" - } + def pathFromModule(relativePath: String): String = + if (sparkDefaultVersion == sparkVersion) { + relativePath + } else { + s"../../spark/$relativePath" + } Project(id = s"spark${sparkVersion.head}", base = file(baseFolder)) .enablePlugins(AssemblyPlugin) .settings( commonSettings, - Compile / scalaSource := baseDirectory.value / pathFromModule("src/main/scala"), + Compile / scalaSource := baseDirectory.value / pathFromModule("src/main/scala"), Compile / resourceDirectory := baseDirectory.value / pathFromModule("src/main/resources"), - Test / scalaSource := baseDirectory.value / pathFromModule("src/test/scala"), - Test / resourceDirectory := baseDirectory.value / pathFromModule("src/test/resources"), - Test / parallelExecution := false, - crossScalaVersions := sparkScalaVersions, + Test / scalaSource := baseDirectory.value / pathFromModule("src/test/scala"), + Test / resourceDirectory := baseDirectory.value / pathFromModule("src/test/resources"), + Test / parallelExecution := false, enablingPublishingSettings, coverageConfig, name := s"osm4scala-spark${sparkVersion.head}", - description := "Spark 2 connector for OpenStreetMap Pbf parser.", + description := s"Spark ${sparkVersion.head} connector for OpenStreetMap Pbf parser.", libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion % Provided ), @@ -173,14 +112,20 @@ def generateSparkModule(sparkVersion: String): Project = { } lazy val spark2 = generateSparkModule(spark2Version) + .settings(crossScalaVersions := Seq(scala212)) + lazy val spark2FatShaded = generateSparkFatShadedModule(spark2Version, spark2) -lazy val spark3 = generateSparkModule(spark3Version) -lazy val spark3FatShaded = generateSparkFatShadedModule(spark3Version, spark3) + .settings(crossScalaVersions := Seq(scala212)) +lazy val spark3 = generateSparkModule(spark3Version) + .settings(crossScalaVersions := Seq(scala212, scala213)) -def listOfProjects(): Seq[ProjectReference] = { +lazy val spark3FatShaded = generateSparkFatShadedModule(spark3Version, spark3) + .settings(crossScalaVersions := Seq(scala212, scala213)) - val modules: Seq[ProjectReference] = Seq( +lazy val root = (project in file(".")) + .disablePlugins(AssemblyPlugin) + .aggregate( core, spark2, spark2FatShaded, @@ -191,27 +136,15 @@ def listOfProjects(): Seq[ProjectReference] = { examplesTagsExtraction, examplesPrimitivesExtraction, examplesBlocksExtraction, - examplesTakeN - ) - - val spark3Projects: Seq[ProjectReference] = Seq( + examplesTakeN, spark3, spark3FatShaded, exampleSparkUtilities, exampleSparkDocumentation ) - - val projects = modules ++ (if(isPatch211Enable()) Seq.empty else spark3Projects) - - println(s"PATCH_211 is ${isPatch211Enable()} so we are going to work with this list of projects: \n${projects.mkString("\t- ", "\n\t- ", "")}") - - projects -} - -lazy val root = (project in file(".")) - .disablePlugins(AssemblyPlugin) - .aggregate( listOfProjects(): _*) .settings( + // crossScalaVersions must be set to Nil on the aggregating project + crossScalaVersions := Nil, name := "osm4scala-root", sonatypeProfileName := "com.acervera.osm4scala", // crossScalaVersions must be set to Nil on the aggregating project @@ -219,7 +152,7 @@ lazy val root = (project in file(".")) publish / skip := true, // don't use sbt-release's cross facility releaseCrossBuild := false, - releaseProcess := Seq[ReleaseStep]( + releaseProcess := Seq[ReleaseStep]( checkSnapshotDependencies, inquireVersions, runClean, @@ -236,6 +169,7 @@ lazy val root = (project in file(".")) lazy val core = Project(id = "core", base = file("core")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, enablingPublishingSettings, coverageConfig, @@ -252,6 +186,7 @@ lazy val core = Project(id = "core", base = file("core")) lazy val commonUtilities = Project(id = "examples-common-utilities", base = file("examples/common-utilities")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, publish / skip := true, @@ -265,6 +200,7 @@ lazy val examplesCounter = Project(id = "examples-counter", base = file("examples/counter")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-counter", @@ -275,6 +211,7 @@ lazy val examplesCounter = lazy val examplesCounterParallel = Project(id = "examples-counter-parallel", base = file("examples/counter-parallel")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-counter-parallel", @@ -285,6 +222,7 @@ lazy val examplesCounterParallel = Project(id = "examples-counter-parallel", bas lazy val examplesCounterAkka = Project(id = "examples-counter-akka", base = file("examples/counter-akka")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-counter-akka", @@ -298,6 +236,7 @@ lazy val examplesCounterAkka = Project(id = "examples-counter-akka", base = file lazy val examplesTagsExtraction = Project(id = "examples-tag-extraction", base = file("examples/tagsextraction")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-tags-extraction", @@ -308,6 +247,7 @@ lazy val examplesTagsExtraction = Project(id = "examples-tag-extraction", base = lazy val examplesBlocksExtraction = Project(id = "examples-blocks-extraction", base = file("examples/blocksextraction")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-blocks-extraction", @@ -315,19 +255,22 @@ lazy val examplesBlocksExtraction = Project(id = "examples-blocks-extraction", b ) .dependsOn(core, commonUtilities) -lazy val examplesBlocksWithIdExtraction = Project(id = "examples-blocks-with-id-extraction", base = file("examples/blockswithidextraction")) - .disablePlugins(AssemblyPlugin) - .settings( - commonSettings, - exampleSettings, - name := "examples-blocks-with-id-extraction", - description := "Extract blocks that contains the id from the pbf into a folder using osm4scala." - ) - .dependsOn(core, commonUtilities) +lazy val examplesBlocksWithIdExtraction = + Project(id = "examples-blocks-with-id-extraction", base = file("examples/blockswithidextraction")) + .disablePlugins(AssemblyPlugin) + .settings( + crossScalaVersions := scalaAllVersions, + commonSettings, + exampleSettings, + name := "examples-blocks-with-id-extraction", + description := "Extract blocks that contains the id from the pbf into a folder using osm4scala." + ) + .dependsOn(core, commonUtilities) lazy val examplesTakeN = Project(id = "examples-takeN", base = file("examples/takeN")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-takeN", @@ -339,6 +282,7 @@ lazy val examplesPrimitivesExtraction = Project(id = "examples-primitives-extraction", base = file("examples/primitivesextraction")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, name := "osm4scala-examples-primitives-extraction", @@ -346,14 +290,12 @@ lazy val examplesPrimitivesExtraction = ) .dependsOn(core, commonUtilities) - - lazy val exampleSparkUtilities = Project(id = "examples-spark-utilities", base = file("examples/spark-utilities")) .disablePlugins(AssemblyPlugin) .settings( + crossScalaVersions := scalaAllVersions, commonSettings, exampleSettings, - crossScalaVersions := Seq(scala212), name := "osm4scala-examples-spark-utilities", description := "Example of different utilities using osm4scala and Spark.", libraryDependencies ++= Seq( @@ -362,17 +304,17 @@ lazy val exampleSparkUtilities = Project(id = "examples-spark-utilities", base = ) .dependsOn(spark3, commonUtilities) - -lazy val exampleSparkDocumentation = Project(id = "examples-spark-documentation", base = file("examples/spark-documentation")) - .disablePlugins(AssemblyPlugin) - .settings( - commonSettings, - exampleSettings, - crossScalaVersions := Seq(scala212), - name := "osm4scala-examples-spark-documentation", - description := "Examples used in the documentation.", - libraryDependencies ++= Seq( - "org.apache.spark" %% "spark-sql" % spark3Version +lazy val exampleSparkDocumentation = + Project(id = "examples-spark-documentation", base = file("examples/spark-documentation")) + .disablePlugins(AssemblyPlugin) + .settings( + crossScalaVersions := scalaAllVersions, + commonSettings, + exampleSettings, + name := "osm4scala-examples-spark-documentation", + description := "Examples used in the documentation.", + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-sql" % spark3Version + ) ) - ) - .dependsOn(spark3, commonUtilities) + .dependsOn(spark3, commonUtilities) diff --git a/project/CommonSettings.scala b/project/CommonSettings.scala new file mode 100644 index 0000000..964765e --- /dev/null +++ b/project/CommonSettings.scala @@ -0,0 +1,82 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2023 Ángel Cervera Claudio + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +import Dependencies._ +import com.jsuereth.sbtpgp.SbtPgp.autoImport.usePgpKeyHex +import sbt.Keys._ +import sbt._ +import sbtassembly.AssemblyKeys.assembly +import xerial.sbt.Sonatype.autoImport.sonatypePublishToBundle + +object CommonSettings { + + lazy val commonSettings = Seq( + organization := "com.acervera.osm4scala", + organizationHomepage := Some(url("https://www.acervera.com")), + licenses += ("MIT", url("https://opensource.org/licenses/MIT")), + ThisBuild / homepage := Some( + url(s"https://simplexspatial.github.io/osm4scala/") + ), + ThisBuild / scmInfo := Some( + ScmInfo( + url("https://github.com/simplexspatial/osm4scala"), + "scm:git:git://github.com/simplexspatial/osm4scala.git", + "scm:git:ssh://github.com:simplexspatial/osm4scala.git" + ) + ), + ThisBuild / developers := List( + Developer( + "angelcervera", + "Angel Cervera Claudio", + "angelcervera@silyan.com", + url("https://www.acervera.com") + ) + ), + libraryDependencies ++= Seq( + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.scalacheck" %% "scalacheck" % scalacheckVersion % Test, + "commons-io" % "commons-io" % commonIOVersion % Test + ), + assembly / test := {}, + scalacOptions ++= Seq( + "-target:jvm-1.8", + "-encoding", + "utf8", + "-deprecation", + "-unchecked", + "-Xlint" + ), + javacOptions ++= Seq( + "-Xlint:all", + "-source", + "1.8", + "-target", + "1.8", + "-parameters" + ), + usePgpKeyHex("A047A2C5A9AFE4850537A00DFC14CE4C2E7B7CBB"), + publishTo := sonatypePublishToBundle.value + ) +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 0000000..5afe677 --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,35 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2023 Ángel Cervera Claudio + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +object Dependencies { + lazy val scalatestVersion = "3.2.0" + lazy val scalacheckVersion = "1.14.3" + lazy val commonIOVersion = "2.5" + lazy val logbackVersion = "1.1.7" + lazy val scoptVersion = "3.7.1" + lazy val akkaVersion = "2.5.31" + lazy val spark3Version = "3.3.1" + lazy val spark2Version = "2.4.8" +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 6d8b188..a8a0c90 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,36 +1,40 @@ -val scalapbVersion = "0.10.2" -val scalapbVersionPatch = "0.9.7" +/* + * The MIT License (MIT) + * + * Copyright (c) 2021 Ángel Cervera Claudio + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ -/** - * From version 0.9.7, ScalaPB drop Scala 2.11 comp. - * I did the same at osm4scala 1.0.4, but at the moment, 90% of Cloud Spark providers - * offers Spark services based on Scala 2.11. - * - * This is a patch that will allow to publish a Scala 2.11 version checking the ENV variable `PATCH_2_11`. - * If `PATCH_211` is `true`, it will use scalapb 0.9.7. Otherwise, it will use the last version. - * - * @return Version to use. - */ -def getScalaPBVersion(): String = sys.env.getOrElse("PATCH_211", "false").toBoolean match { - case true => scalapbVersionPatch - case false => scalapbVersion -} +addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.34") +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.10.2" addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.7") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("com.github.sbt" % "sbt-release" % "1.0.15") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") -addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.34") - -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.2.7") - -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % getScalaPBVersion() logLevel := Level.Warn -// Helpers for dev time. addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.20") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") +addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.2.7")