Source build now relies on pkg-config

sources/default.nix

@@ -1,5 +1,5 @@
 { stdenv, boost, openssl, lowdown, nlohmann_json, brotli, libsodium, editline
-, gnutar, coreutils, findutils, python3, nix
+, gnutar, coreutils, findutils, python3, nix, libarchive
 , automake, autoconf-archive, autoconf, m4, bc, libtool, pkg-config
 # External source for Nix
 , nix-source ? nix.src
@@ -56,6 +56,10 @@ let
       '';
       dest = "libbrotlicommon";
     };
+  libarchive_configured_src = mkConfiguredSrc
+    { pkg = libarchive;
+      confScript = "./build/autogen.sh";
+    };
 
   srcs_simple =
     [ openssl
@@ -69,6 +73,7 @@ let
     [ nix_configured_src
       editline_configured_src
       brotli_configured_src
+      libarchive_configured_src
     ];
 in stdenv.mkDerivation {
   name = "nixie-sources";

src/builders.ab

@@ -20,6 +20,8 @@ import { build_autoconf_dep }   from "./builders/autoconf.ab"
 import { build_nix }            from "./builders/nix.ab"
 
 /// Export libraries found inside the macOS SDK for building Nix
+///
+/// TODO: is this still required with Meson?
 fun darwin_export_sdk()
 {
     // Calling xcrun should prompt the user to install the macOS SDK.
@@ -46,8 +48,8 @@ fun darwin_export_sdk()
 /// Build Nix and its dependencies locally, then place it in the expected location
 /// in the user's cache directory.
 ///
-/// This process **requires**, among other things, `pkg-config` to determine
-/// which dependencies are already met and avoid building them.
+/// This process **requires**, among other things, `pkg-config` due to it being
+/// the only detection method for many dependencies in Meson.
 pub fun try_build_nix()
 {
     let cache_root = get_cache_root()
@@ -58,18 +60,19 @@ pub fun try_build_nix()
     if get_osname() == "Darwin":
         darwin_export_sdk()
 
-    trust env_var_set("step_total", "8")
+    trust env_var_set("step_total", "9")
 
     dir_create(get_source_root())
-    dir_create("{cache_root}/nix-lib")
+    dir_create("{cache_root}/nix-deps/lib/pkgconfig")
 
     build_openssl()?
     build_boost()?
     build_nlohmann_json()?
     build_lowdown()?
-    //                 pkgconf name       env var       lib prefix       include prefix
-    build_autoconf_dep("libbrotlicommon", "LIBBROTLI",  "",              "c/include")?
-    build_autoconf_dep("libsodium",       "SODIUM",     "src/libsodium", "src/libsodium/include")?
-    build_autoconf_dep("libeditline",     "EDITLINE",   "src",           "include")?
+    //                 pkgconf name       include prefix
+    build_autoconf_dep("libbrotlicommon", "c/include")?
+    build_autoconf_dep("libsodium",       "src/libsodium/include")?
+    build_autoconf_dep("libeditline",     "include")?
+    build_autoconf_dep("libarchive",      "libarchive")?
     build_nix()?
 }

src/builders/autoconf.ab

@@ -13,12 +13,8 @@ import { pkg_exists, step_title, get_source_root } from "./common.ab"
 ///
 /// ### Arguments:
 /// - `lib_name`: The library to check for with `pkg-config`
-/// - `var_name`: The variable to export library locations
-/// - `lib_prefix`: Where in the source tree the resulting libraries are found
 /// - `inc_prefix`: Where in the source tree the build headers are found
-pub fun build_autoconf_dep(lib_name: Text, var_name: Text,
-                           lib_prefix: Text = "",
-                           inc_prefix: Text = ""): Null?
+pub fun build_autoconf_dep(lib_name: Text, inc_prefix: Text = ""): Null?
 {
     let source_root = get_source_root()
     let cache_root = get_cache_root()
@@ -32,34 +28,25 @@ pub fun build_autoconf_dep(lib_name: Text, var_name: Text,
 
     pull_source_file(lib_name, my_source)?
 
-    $(unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH && cd {my_source} && ./configure && make)$?
-
-    $cp {my_source}/{lib_prefix}/.libs/* {cache_root}/nix-lib/$?
-    trust $rm {cache_root}/nix-lib/*.o$
-
-    trust env_var_set("{var_name}_LIBS", "{cache_root}/nix-lib")
-    trust env_var_set("{var_name}_CFLAGS", "-I{my_source}/{inc_prefix}/include")
-    trust $export {var_name}_LIBS {var_name}_CFLAGS$
-
-    trust $export C_INCLUDE_PATH={my_source}/{inc_prefix}:\$C_INCLUDE_PATH$
-    trust $export CPLUS_INCLUDE_PATH={my_source}/{inc_prefix}:\$CPLUS_INCLUDE_PATH$
+    $( unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH \
+    && cd {my_source} \
+    && ./configure --prefix={cache_root}/nix-deps \
+    && make && make install )$?
 }
 
 main(cmdl)
 {
     if len(cmdl) < 5 {
-        echo "Usage: ./autoconf.sh <package> <var_name> <lib_prefix> <inc_prefix>"
+        echo "Usage: ./autoconf.sh <package> <inc_prefix>"
         echo ""
         echo "See builders/autoconf.ab and builders.ab for more info"
         exit 1
     }
 
     let lib_name = cmdl[1]
-    let var_name = cmdl[2]
-    let lib_prefix = cmdl[3]
-    let inc_prefix = cmdl[4]
+    let inc_prefix = cmdl[2]
 
     trust env_var_set("_NIXIE_TESTING_SKIP_TARBALL", "1")
     trust env_var_set("step_total", "1")
-    build_autoconf_dep(lib_name, var_name, lib_prefix, inc_prefix)?
+    build_autoconf_dep(lib_name, inc_prefix)?
 }

src/builders/lowdown.ab

@@ -18,11 +18,14 @@ fun macos_build_post()
 /// This function runs within the source directory, in a subshell.
 fun build_lowdown_inner()
 {
-    $./configure$?
+    let cache_root = get_cache_root()
+
+    $./configure PREFIX={cache_root}/nix-deps$?
     $make$?
 
     if get_osname() == "Darwin":
         macos_build_post()?
+    $make install_shared$?
 }
 
 /// This is the core function for the Lowdown builder.
@@ -41,12 +44,6 @@ pub fun build_lowdown()
     pull_source_file("lowdown", "{source_root}/lowdown")?
 
     $(cd {source_root}/lowdown && {nameof build_lowdown_inner})$?
-
-    trust env_var_set("LOWDOWN_LIBS", "{cache_root}/nix-lib")
-    trust env_var_set("LOWDOWN_CFLAGS", "-I{source_root}/lowdown")
-
-    trust $export C_INCLUDE_PATH={source_root}/lowdown:\$C_INCLUDE_PATH$
-    trust $export CPLUS_INCLUDE_PATH={source_root}/lowdown:\$CPLUS_INCLUDE_PATH$
 }
 
 main(cmdl)

src/builders/nix.ab

@@ -2,6 +2,7 @@
 // Licensed under GNU GPLv2
 // Builder for the Nix package manager itself
 
+import { env_var_set } from "std/env"
 import { file_exists, dir_exists, dir_create } from "std/fs"
 
 import { pull_source_file } from "../resources.ab"
@@ -46,7 +47,8 @@ pub fun build_nix()
 
     $python3 -m venv --system-site-packages "{venv}"$?
 
-    trust $export LIBRARY_PATH={cache_root}/nix-lib:\$LIBRARY_PATH$
+    trust $export LIBRARY_PATH={cache_root}/nix-deps/lib:\$LIBRARY_PATH$
+    trust $export PKG_CONFIG_PATH={cache_root}/nix-deps/lib/pkgconfig:{cache_root}/nix-deps/share/pkgconfig:\$PKG_CONFIG_PATH$
 
     ${venv}/bin/pip install meson ninja$?
 
@@ -57,10 +59,7 @@ pub fun build_nix()
 
 main(cmdl)
 {
-    echo "This builder only makes sense in the wrapper context,"
-    echo "where all dependencies were built and/or resolved."
-    echo ""
-    echo "Use ./nix --nixie-no-precompiled to test the full build chain."
-
-    exit 1
+    trust env_var_set("_nixie_testing_skip_tarball", "1")
+    trust env_var_set("step_total", "1")
+    build_nix()?
 }

src/builders/nlohmann_json.ab

@@ -3,17 +3,18 @@
 // Builder for NLohmann's JSON library
 
 import { env_var_set } from "std/env"
+import { file_write } from "std/fs"
 
 import { pull_source_file } from "../resources.ab"
+import { get_cache_root } from "../platform.ab"
 
 import { pkg_exists, step_title, get_source_root } from "./common.ab"
 
 /// This is the core function for the NLohmann JSON library.
-///
-/// It exports the `LNLOHMANN_JSON_LIBS` and `NLOHMANN_JSON_CFLAGS` variables.
 pub fun build_nlohmann_json()
 {
     let source_root = get_source_root()
+    let cache_root = get_cache_root()
 
     step_title("nlohmann_json")
 
@@ -22,12 +23,13 @@ pub fun build_nlohmann_json()
 
     pull_source_file("nlohmann_json", "{source_root}/nlohmann_json")?
 
-    trust env_var_set("NLOHMANN_JSON_LIBS", "{source_root}/nlohmann_json/single_include")
-    trust env_var_set("NLOHMANN_JSON_CFLAGS", "{source_root}/nlohmann_json/single_include")
-    trust $export NLOHMANN_JSON_LIBS NLOHMANN_JSON_CFLAGS$
+    let version = trust $grep "^version:" {source_root}/nlohmann_json/wsjcpp.yml | cut -d '"' -f 2 | cut -d 'v' -f 2$
 
-    trust $export C_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$C_INCLUDE_PATH$
-    trust $export CPLUS_INCLUDE_PATH={source_root}/nlohmann_json/single_include:\$CPLUS_INCLUDE_PATH$
+    file_write("{cache_root}/nix-deps/lib/pkgconfig/nlohmann_json.pc",
+"Name: nlohmann_json
+Version: {version}
+Description: JSON for Modern C++
+Cflags: -I{source_root}/nlohmann_json/include")?
 }
 
 main(cmdl)

src/builders/openssl.ab

@@ -31,9 +31,16 @@ fun build_openssl_inner()
 
     $./config$?
     make_headers()?
-    $make libcrypto.{dll_ext}$?
-
-    trust $cp ./libcrypto.* {cache_root}/nix-lib/$
+    $make libcrypto.{dll_ext} libcrypto.pc$?
+
+    // Building libssl is:
+    // - hard
+    // - broken
+    // - a very bad idea anyway
+    // so we're performing the install ourselves
+    trust $cp ./libcrypto.* {cache_root}/nix-deps/lib/$
+    trust $cp ./libcrypto.pc {cache_root}/nix-deps/lib/pkgconfig$
+    trust $cp -r ./include {cache_root}/nix-deps/$
 }
 
 /// This is the core function for the OpenSSL builder.
@@ -55,13 +62,6 @@ pub fun build_openssl()
 
     // Using a subshell ensures we aren't cd elsewhere on failure
     $(cd {source_root}/openssl && {nameof build_openssl_inner})$?
-
-    trust env_var_set("OPENSSL_LIBS", "{cache_root}/nix-lib")
-    trust env_var_set("OPENSSL_CFLAGS", "-I{source_root}/openssl/include")
-    trust $export OPENSSL_LIBS OPENSSL_CFLAGS$
-
-    trust $export C_INCLUDE_PATH={source_root}/openssl/include:\$C_INCLUDE_PATH$
-    trust $export CPLUS_INCLUDE_PATH={source_root}/openssl/include:\$CPLUS_INCLUDE_PATH$
 }
 
 main(cmdl)

src/cli.ab

@@ -56,7 +56,7 @@ fun cmd_cleanup()
     trust $rm -rf {nix_root}$
 
     echo "Removing retrieved Nix binaries..."
-    trust $rm -rf {cache_root}/nix-static {cache_root}/nix-lib$
+    trust $rm -rf {cache_root}/nix-static {cache_root}/nix-lib {cache_root}/nix-deps$
 
     exit 0
 }

src/nix.ab

@@ -46,7 +46,7 @@ fun get_nix()
     let system = get_system()
 
     let nix_path = "{cache_root}/nix-static"
-    let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib"
+    let fakedir_path = "{cache_root}/nix-deps/lib/libfakedir.dylib"
 
     enter_alt_buffer()
     set_title("Building Nix...")
@@ -56,7 +56,7 @@ fun get_nix()
 
     // Unpack fakedir as needed
     if osname == "Darwin" and not file_exists(fakedir_path) {
-        dir_create("{cache_root}/nix-lib")
+        dir_create("{cache_root}/nix-deps/lib")
         pull_binary("libfakedir.dylib", fakedir_path) failed {
             teardown(true)
             fail 1
@@ -176,7 +176,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null
     let cache_root = get_cache_root()
     let nix_root = get_nix_root()
 
-    let fakedir_path = "{cache_root}/nix-lib/libfakedir.dylib"
+    let fakedir_path = "{cache_root}/nix-deps/lib/libfakedir.dylib"
 
     trust env_var_set("FAKEDIR_PATTERN", "/nix")
     trust env_var_set("FAKEDIR_TARGET", "{nix_root}/nix")
@@ -187,7 +187,7 @@ fun launch_darwin_workaround(name: Text, nix_path: Text, args: [Text]): Null
     // Unfortunately, this requires us to disable the Nix sandbox entirely.
     trust $ _NIX_TEST_NO_SANDBOX=1 \
             DYLD_INSERT_LIBRARIES="{fakedir_path}" \
-            DYLD_LIBRARY_PATH="{cache_root}/nix-lib" \
+            DYLD_LIBRARY_PATH="{cache_root}/nix-deps/lib" \
                 exec -a {name} {nix_path} "{args}"$
 }
 

src/resources.ab

@@ -64,6 +64,9 @@ pub fun pull_source_file(member: Text, dest: Text): Null?
         where = "{tmpd}/{member}"
     }
 
+    // Our build method means source dirs accumulate state, we're better off
+    // throwing them away.
+    trust $rm -rf {dest}$
     trust mv where dest
 }