Default delta updater can delta non-block resources

Requires fwup >= 1.10.0

Closes #885

apps/nerves_hub_web_core/lib/nerves_hub_web_core/devices.ex

@@ -23,7 +23,7 @@ defmodule NervesHubWebCore.Devices do
   alias NervesHubWebCore.Devices.{Device, DeviceCertificate, CACertificate}
   alias NervesHubWebCore.TaskSupervisor, as: Tasks
 
-  @min_fwup_delta_updatable_version ">=1.6.0"
+  @min_fwup_delta_updatable_version ">=1.10.0"
 
   def get_device(device_id), do: Repo.get(Device, device_id)
   def get_device!(device_id), do: Repo.get!(Device, device_id)

apps/nerves_hub_web_core/lib/nerves_hub_web_core/firmwares/delta_updater/default.ex

@@ -31,45 +31,106 @@ defmodule NervesHubWebCore.Firmwares.DeltaUpdater.Default do
       )
 
     output_filename = uuid <> ".fw"
-    output = Path.join(work_dir, output_filename) |> Path.expand()
+    output_path = Path.join(work_dir, output_filename) |> Path.expand()
+
+    do_delta_file(source_path, target_path, output_path, work_dir)
+  end
+
+  @impl NervesHubWebCore.Firmwares.DeltaUpdater
+  def cleanup_firmware_delta_files(firmware_delta_path) do
+    firmware_delta_path
+    |> Path.dirname()
+    |> File.rm_rf!()
+
+    :ok
+  end
+
+  @impl NervesHubWebCore.Firmwares.DeltaUpdater
+  def delta_updatable?(file_path) do
+    {meta, 0} = System.cmd("unzip", ["-qqp", file_path, "meta.conf"])
+
+    (meta =~ "delta-source-raw-offset" && meta =~ "delta-source-raw-count") or
+      (meta =~ "delta-source-fat-offset" && meta =~ "delta-source-fat-path")
+  end
+
+  def do_delta_file(source_path, target_path, output_path, work_dir) do
+    File.mkdir_p(work_dir)
 
     source_work_dir = Path.join(work_dir, "source")
     target_work_dir = Path.join(work_dir, "target")
     output_work_dir = Path.join(work_dir, "output")
 
     File.mkdir_p(source_work_dir)
     File.mkdir_p(target_work_dir)
-    File.mkdir_p(Path.join(output_work_dir, "data"))
+    File.mkdir_p(output_work_dir)
 
     {_, 0} = System.cmd("unzip", ["-qq", source_path, "-d", source_work_dir])
     {_, 0} = System.cmd("unzip", ["-qq", target_path, "-d", target_work_dir])
 
-    source_rootfs = Path.join([source_work_dir, "data", "rootfs.img"])
-    target_rootfs = Path.join([target_work_dir, "data", "rootfs.img"])
-    out_rootfs = Path.join([output_work_dir, "data", "rootfs.img"])
+    for path <- Path.wildcard(target_work_dir <> "/**") do
+      path = Regex.replace(~r/^#{target_work_dir}\//, path, "")
+
+      unless File.dir?(Path.join(target_work_dir, path)) do
+        :ok = handle_content(path, source_work_dir, target_work_dir, output_work_dir)
+      end
+    end
+
+    # firmware archive files order matters:
+    # 1. meta.conf.ed25519 (optional)
+    # 2. meta.conf
+    # 3. other...
+    [
+      "meta.conf.*",
+      "meta.conf",
+      "data"
+    ]
+    |> Enum.each(&add_to_zip(&1, output_work_dir, output_path))
+
+    output_path
+  end
 
-    {_, 0} =
-      System.cmd("xdelta3", ["-A", "-S", "-f", "-s", source_rootfs, target_rootfs, out_rootfs])
+  defp handle_content("meta." <> _ = path, _source_dir, target_dir, out_dir) do
+    do_copy(Path.join(target_dir, path), Path.join(out_dir, path))
+  end
 
-    File.mkdir_p!(Path.dirname(output))
-    File.cp!(target_path, output)
+  defp handle_content(path, source_dir, target_dir, out_dir) do
+    do_delta(Path.join(source_dir, path), Path.join(target_dir, path), Path.join(out_dir, path))
+  end
 
-    {_, 0} = System.cmd("zip", ["-qq", output, "data/rootfs.img"], cd: output_work_dir)
-    output
+  defp do_copy(source, target) do
+    target |> Path.dirname() |> File.mkdir_p!()
+    File.cp(source, target)
   end
 
-  @impl NervesHubWebCore.Firmwares.DeltaUpdater
-  def cleanup_firmware_delta_files(firmware_delta_path) do
-    firmware_delta_path
-    |> Path.dirname()
-    |> File.rm_rf!()
+  defp do_delta(source, target, out) do
+    out |> Path.dirname() |> File.mkdir_p!()
 
-    :ok
+    with {_, 0} <-
+           System.cmd("xdelta3", ["-A", "-S", "-f", "-s", source, target, out]) do
+      :ok
+    else
+      {_, code} ->
+        {:error, code}
+    end
   end
 
-  @impl NervesHubWebCore.Firmwares.DeltaUpdater
-  def delta_updatable?(file_path) do
-    {meta, 0} = System.cmd("unzip", ["-qqp", file_path, "meta.conf"])
-    meta =~ "delta-source-raw-offset" && meta =~ "delta-source-raw-count"
+  defp add_to_zip(glob, workdir, output) do
+    workdir
+    |> Path.join(glob)
+    |> Path.wildcard()
+    |> case do
+      [] ->
+        :ok
+
+      paths ->
+        {_, 0} =
+          System.cmd(
+            "zip",
+            ["-r", "-qq", output | Enum.map(paths, &Path.relative_to(&1, workdir))],
+            cd: workdir
+          )
+
+        :ok
+    end
   end
 end

apps/nerves_hub_web_core/test/nerves_hub_web_core/devices/devices_test.exs

@@ -16,7 +16,7 @@ defmodule NervesHubWebCore.DevicesTest do
   alias NervesHubWebCore.Devices.{DeviceCertificate, UpdatePayload}
   alias Ecto.Changeset
 
-  @valid_fwup_version "1.6.0"
+  @valid_fwup_version "1.10.0"
 
   setup do
     user = Fixtures.user_fixture()
@@ -616,7 +616,7 @@ defmodule NervesHubWebCore.DevicesTest do
 
       deployment = Fixtures.deployment_fixture(org, target, %{name: "resolve-update"})
       device = Fixtures.device_fixture(org, product, source)
-      {:ok, device} = Devices.update_firmware_metadata(device, %{fwup_version: "1.6.0"})
+      {:ok, device} = Devices.update_firmware_metadata(device, %{fwup_version: "1.10.0"})
       %{firmware_metadata: %{fwup_version: fwup_version}} = device
 
       firmware_delta = Fixtures.firmware_delta_fixture(source, target)
@@ -642,7 +642,7 @@ defmodule NervesHubWebCore.DevicesTest do
 
       deployment = Fixtures.deployment_fixture(org, target, %{name: "resolve-update"})
       device = Fixtures.device_fixture(org, product, source)
-      {:ok, device} = Devices.update_firmware_metadata(device, %{fwup_version: "1.6.0"})
+      {:ok, device} = Devices.update_firmware_metadata(device, %{fwup_version: "1.10.0"})
       %{firmware_metadata: %{fwup_version: fwup_version}} = device
 
       assert Devices.delta_updatable?(source, target, deployment, fwup_version)