Implement file management (#2022)

Co-authored-by: José Valim <jose.valim@dashbit.co>

assets/css/js_interop.css

@@ -224,6 +224,11 @@ solely client-side operations.
   @apply hidden;
 }
 
+[data-el-session]:not([data-js-side-panel-content="files-list"])
+  [data-el-files-list] {
+  @apply hidden;
+}
+
 [data-el-session]:not([data-js-side-panel-content="runtime-info"])
   [data-el-runtime-info] {
   @apply hidden;
@@ -249,6 +254,11 @@ solely client-side operations.
   @apply text-gray-50 bg-gray-700;
 }
 
+[data-el-session][data-js-side-panel-content="files-list"]
+  [data-el-files-list-toggle] {
+  @apply text-gray-50 bg-gray-700;
+}
+
 [data-el-session][data-js-side-panel-content="runtime-info"]
   [data-el-runtime-info-toggle] {
   @apply text-gray-50 bg-gray-700;

assets/js/hooks/session.js

@@ -121,6 +121,10 @@ const Session = {
       this.toggleAppInfo()
     );
 
+    this.getElement("files-list-toggle").addEventListener("click", (event) =>
+      this.toggleFilesList()
+    );
+
     this.getElement("notebook").addEventListener("scroll", (event) =>
       this.updateSectionListHighlight()
     );
@@ -368,6 +372,8 @@ const Session = {
         this.toggleAppInfo();
       } else if (keyBuffer.tryMatch(["s", "u"])) {
         this.toggleClientsList();
+      } else if (keyBuffer.tryMatch(["s", "f"])) {
+        this.toggleFilesList();
       } else if (keyBuffer.tryMatch(["s", "r"])) {
         this.toggleRuntimeInfo();
       } else if (keyBuffer.tryMatch(["s", "b"])) {
@@ -719,6 +725,10 @@ const Session = {
     this.toggleSidePanelContent("app-info");
   },
 
+  toggleFilesList() {
+    this.toggleSidePanelContent("files-list");
+  },
+
   toggleRuntimeInfo() {
     this.toggleSidePanelContent("runtime-info");
   },

lib/livebook/file_system/s3.ex

@@ -30,8 +30,11 @@ defmodule Livebook.FileSystem.S3 do
     bucket_url = String.trim_trailing(bucket_url, "/")
     region = opts[:region] || region_from_uri(bucket_url)
 
+    hash = :crypto.hash(:sha256, bucket_url) |> Base.url_encode64(padding: false)
+    id = "s3-#{hash}"
+
     %__MODULE__{
-      id: Livebook.Utils.random_short_id(),
+      id: id,
       bucket_url: bucket_url,
       region: region,
       access_key_id: access_key_id,
@@ -52,23 +55,22 @@ defmodule Livebook.FileSystem.S3 do
   def from_config(config) do
     case config do
       %{
-        id: id,
         bucket_url: bucket_url,
         access_key_id: access_key_id,
         secret_access_key: secret_access_key
       } ->
         file_system = new(bucket_url, access_key_id, secret_access_key, region: config[:region])
-        {:ok, %{file_system | id: id}}
+        {:ok, file_system}
 
       _config ->
         {:error,
-         "S3 file system config is expected to have keys: :id, :bucket_url, :access_key_id and :secret_access_key, but got #{inspect(config)}"}
+         "S3 file system config is expected to have keys: :bucket_url, :access_key_id and :secret_access_key, but got #{inspect(config)}"}
     end
   end
 
   @spec to_config(t()) :: map()
   def to_config(%__MODULE__{} = s3) do
-    Map.take(s3, [:id, :bucket_url, :region, :access_key_id, :secret_access_key])
+    Map.take(s3, [:bucket_url, :region, :access_key_id, :secret_access_key])
   end
 end
 

lib/livebook/live_markdown/export.ex

@@ -82,11 +82,16 @@ defmodule Livebook.LiveMarkdown.Export do
 
     app_settings_metadata = app_settings_metadata(notebook.app_settings)
 
-    if app_settings_metadata == %{} do
-      metadata
-    else
-      Map.put(metadata, :app_settings, app_settings_metadata)
-    end
+    file_entry_metadatas =
+      notebook.file_entries
+      |> Enum.sort_by(& &1.name)
+      |> Enum.map(&file_entry_metadata/1)
+
+    put_unless_default(
+      metadata,
+      %{app_settings: app_settings_metadata, file_entries: file_entry_metadatas},
+      %{app_settings: %{}, file_entries: []}
+    )
   end
 
   defp app_settings_metadata(app_settings) do
@@ -108,6 +113,18 @@ defmodule Livebook.LiveMarkdown.Export do
     )
   end
 
+  defp file_entry_metadata(%{type: :attachment, name: name}) do
+    %{type: "attachment", name: name}
+  end
+
+  defp file_entry_metadata(%{type: :file, name: name, file: file}) do
+    %{type: "file", name: name, file: %{file_system_id: file.file_system.id, path: file.path}}
+  end
+
+  defp file_entry_metadata(%{type: :url, name: name, url: url}) do
+    %{type: "url", name: name, url: url}
+  end
+
   defp render_section(section, notebook, ctx) do
     name = ["## ", section.name]
 
@@ -362,13 +379,16 @@ defmodule Livebook.LiveMarkdown.Export do
     put_unless_default(%{}, Map.take(notebook, keys), Map.take(Notebook.new(), keys))
   end
 
-  defp ensure_order(map) do
+  defp ensure_order(%{} = map) do
     map
     |> Enum.sort()
-    |> Enum.map(fn
-      {key, %{} = value} -> {key, ensure_order(value)}
-      pair -> pair
-    end)
+    |> Enum.map(fn {key, value} -> {key, ensure_order(value)} end)
     |> Jason.OrderedObject.new()
   end
+
+  defp ensure_order(list) when is_list(list) do
+    Enum.map(list, &ensure_order/1)
+  end
+
+  defp ensure_order(term), do: term
 end

lib/livebook/live_markdown/import.ex

@@ -381,35 +381,58 @@ defmodule Livebook.LiveMarkdown.Import do
 
   defp notebook_metadata_to_attrs(metadata) do
     Enum.reduce(metadata, {%{}, Livebook.Hubs.Personal.id(), []}, fn
-      {"persist_outputs", persist_outputs}, {attrs, id, messages} ->
-        {Map.put(attrs, :persist_outputs, persist_outputs), id, messages}
+      {"persist_outputs", persist_outputs}, {attrs, stamp_hub_id, messages} ->
+        {Map.put(attrs, :persist_outputs, persist_outputs), stamp_hub_id, messages}
 
-      {"autosave_interval_s", autosave_interval_s}, {attrs, id, messages} ->
-        {Map.put(attrs, :autosave_interval_s, autosave_interval_s), id, messages}
+      {"autosave_interval_s", autosave_interval_s}, {attrs, stamp_hub_id, messages} ->
+        {Map.put(attrs, :autosave_interval_s, autosave_interval_s), stamp_hub_id, messages}
 
-      {"default_language", default_language}, {attrs, id, messages}
+      {"default_language", default_language}, {attrs, stamp_hub_id, messages}
       when default_language in ["elixir", "erlang"] ->
         default_language = String.to_atom(default_language)
-        {Map.put(attrs, :default_language, default_language), id, messages}
+        {Map.put(attrs, :default_language, default_language), stamp_hub_id, messages}
 
-      {"hub_id", hub_id}, {attrs, id, messages} ->
+      {"hub_id", hub_id}, {attrs, stamp_hub_id, messages} ->
         cond do
           Hubs.hub_exists?(hub_id) -> {Map.put(attrs, :hub_id, hub_id), hub_id, messages}
           Hubs.get_offline_hub(hub_id) -> {attrs, hub_id, messages}
-          true -> {attrs, id, messages ++ ["ignoring notebook Hub with unknown id"]}
+          true -> {attrs, stamp_hub_id, messages ++ ["ignoring notebook Hub with unknown id"]}
         end
 
-      {"app_settings", app_settings_metadata}, {attrs, id, messages} ->
+      {"app_settings", app_settings_metadata}, {attrs, stamp_hub_id, messages} ->
         app_settings =
           Map.merge(
             Notebook.AppSettings.new(),
             app_settings_metadata_to_attrs(app_settings_metadata)
           )
 
-        {Map.put(attrs, :app_settings, app_settings), id, messages}
-
-      _entry, {attrs, id, messages} ->
-        {attrs, id, messages}
+        {Map.put(attrs, :app_settings, app_settings), stamp_hub_id, messages}
+
+      {"file_entries", file_entry_metadatas}, {attrs, stamp_hub_id, messages}
+      when is_list(file_entry_metadatas) ->
+        file_system_by_id =
+          if Enum.any?(file_entry_metadatas, &(&1["type"] == "file")) do
+            for file_system <- Livebook.Settings.file_systems(),
+                do: {file_system.id, file_system},
+                into: %{}
+          else
+            %{}
+          end
+
+        {file_entries, file_entry_messages} =
+          for file_entry_metadata <- file_entry_metadatas, reduce: {[], []} do
+            {file_entries, warnings} ->
+              case file_entry_metadata_to_attrs(file_entry_metadata, file_system_by_id) do
+                {:ok, file_entry} -> {[file_entry | file_entries], warnings}
+                {:error, message} -> {file_entries, [message | warnings]}
+              end
+          end
+
+        {Map.put(attrs, :file_entries, file_entries), stamp_hub_id,
+         messages ++ file_entry_messages}
+
+      _entry, {attrs, stamp_hub_id, messages} ->
+        {attrs, stamp_hub_id, messages}
     end)
   end
 
@@ -444,6 +467,33 @@ defmodule Livebook.LiveMarkdown.Import do
     end)
   end
 
+  defp file_entry_metadata_to_attrs(%{"type" => "attachment", "name" => name}, _file_system_by_id) do
+    {:ok, %{type: :attachment, name: name}}
+  end
+
+  defp file_entry_metadata_to_attrs(
+         %{
+           "type" => "file",
+           "name" => name,
+           "file" => %{"file_system_id" => file_system_id, "path" => path}
+         },
+         file_system_by_id
+       ) do
+    if file_system = file_system_by_id[file_system_id] do
+      file = Livebook.FileSystem.File.new(file_system, path)
+      {:ok, %{type: :file, name: name, file: file}}
+    else
+      {:error, "skipping file #{name}, since it points to an unknown file system"}
+    end
+  end
+
+  defp file_entry_metadata_to_attrs(
+         %{"type" => "url", "name" => name, "url" => url},
+         _file_system_by_id
+       ) do
+    {:ok, %{type: :url, name: name, url: url}}
+  end
+
   defp section_metadata_to_attrs(metadata) do
     Enum.reduce(metadata, %{}, fn
       {"branch_parent_index", parent_idx}, attrs ->
@@ -518,7 +568,27 @@ defmodule Livebook.LiveMarkdown.Import do
         end
       end)
 
-    {%{notebook | sections: sections}, Enum.reverse(warnings)}
+    notebook = %{notebook | sections: sections}
+
+    legacy_images? =
+      notebook
+      |> Notebook.cells_with_section()
+      |> Enum.any?(fn {cell, _section} ->
+        # A heuristic to detect legacy image source
+        is_struct(cell, Notebook.Cell.Markdown) and String.contains?(cell.source, "](images/")
+      end)
+
+    image_warnings =
+      if legacy_images? do
+        [
+          "found Markdown images pointing to the images/ directory." <>
+            " Using this directory has been deprecated, please use notebook files instead"
+        ]
+      else
+        []
+      end
+
+    {notebook, Enum.reverse(warnings) ++ image_warnings}
   end
 
   defp take_stamp_data([{:stamp, data} | elements]), do: {data, elements}

lib/livebook/migration.ex

@@ -10,6 +10,7 @@ defmodule Livebook.Migration do
     insert_personal_hub()
     move_app_secrets_to_personal_hub()
     add_personal_hub_secret_key()
+    update_file_systems_to_deterministic_ids()
   end
 
   defp delete_local_host_hub() do
@@ -46,4 +47,44 @@ defmodule Livebook.Migration do
       Livebook.Storage.insert(:hubs, Livebook.Hubs.Personal.id(), secret_key: secret_key)
     end
   end
+
+  defp update_file_systems_to_deterministic_ids() do
+    # We changed S3 file system ids, such that they are deterministic
+    # for the same bucket, rather than random. We take this opportunity
+    # to rename the scope from :filesystem to :file_systems, which
+    # conveniently allows for easy check if there's anything to migrate.
+    # This migration can be removed in the future (at the cost of discarding
+    # very old file systems (which can be re-added).
+    # TODO: remove on Livebook v0.12
+
+    case Livebook.Storage.all(:filesystem) do
+      [] ->
+        :ok
+
+      configs ->
+        id_mapping =
+          for config <- configs, into: %{} do
+            old_id = config.id
+            # At this point S3 is the only file system we store
+            {:ok, file_system} = Livebook.FileSystem.S3.from_config(config)
+            Livebook.Settings.save_file_system(file_system)
+            Livebook.Storage.delete(:filesystem, old_id)
+            {old_id, file_system.id}
+          end
+
+        # Remap default file system id
+
+        default_file_system_id = Livebook.Settings.default_file_system_id()
+
+        if new_id = id_mapping[default_file_system_id] do
+          Livebook.Settings.set_default_file_system(new_id)
+        end
+
+        # Livebook.NotebookManager is started before the migration runs,
+        # and it discards S3 files, since it can't find the file system.
+        # However, in this case it's fine; for recent notebooks it does
+        # not matter really and there are likely not many starred S3
+        # notebooks at this point (and the user can easily star again)
+    end
+  end
 end