1/n: Tramp FileId into erlang_service compile, for merge

Summary:
# Context

This diff stack is about disconnecting the erlang service from the file system. This is done by passing files to be parsed, and include files to be resolved and opened from ELP to the erlang service via IPC.  This avoids race conditions on the disk, and ensures we always process the "source of truth" in the IDE, as required by the LSP data model.

It potentially opens the way for as-you-type erlang service and eqwalizer diagnostcs, should we choose to enable that.

# This Diff

When resolving include files in ELP we need to know the file we are including from, to be able to work out the correct application, and for relative path includes.  ELP tracks files via `FileId`, which is a `u32` index into the salsa data store.

Since resolution is now happening via a callback from the Erlang Service, make the `FileId` available when the initial parse request is made.

This will allow it to be passed back and used in the callback request, later in this stack.

This change simply makes it available in the `elp_epp` state.

Reviewed By: michalmuskala, robertoaloi

Differential Revision: D62189172

fbshipit-source-id: a0e953341800bdb937327de262684e979deed656

crates/erlang_service/Cargo.toml

@@ -4,6 +4,7 @@ edition.workspace = true
 version.workspace = true
 
 [dependencies]
+elp_base_db.workspace = true
 elp_syntax.workspace = true
 
 anyhow.workspace = true

crates/erlang_service/src/lib.rs

@@ -36,6 +36,7 @@ use crossbeam_channel::Receiver;
 use crossbeam_channel::Sender;
 use eetf::pattern;
 use eetf::Term;
+use elp_base_db::FileId;
 use elp_syntax::SmolStr;
 use fxhash::FxHashMap;
 use fxhash::FxHashSet;
@@ -135,6 +136,7 @@ pub enum Format {
 pub struct ParseRequest {
     pub options: Vec<CompileOption>,
     pub override_options: Vec<CompileOption>,
+    pub file_id: FileId,
     pub path: PathBuf,
     pub format: Format,
 }
@@ -762,6 +764,9 @@ impl ParseRequest {
             .collect::<Vec<eetf::Term>>();
         let list = eetf::List::from(vec![
             path_into_list(self.path).into(),
+            eetf::Term::FixInteger(eetf::FixInteger {
+                value: self.file_id.index() as i32,
+            }),
             eetf::List::from(options).into(),
             eetf::List::from(override_options).into(),
         ]);
@@ -986,6 +991,7 @@ mod tests {
         let request = ParseRequest {
             options: vec![],
             override_options,
+            file_id: FileId::from_raw(0),
             path,
             format: Format::Text,
         };
@@ -1010,6 +1016,7 @@ mod tests {
         let request = ParseRequest {
             options: vec![],
             override_options,
+            file_id: FileId::from_raw(0),
             path,
             format: Format::Text,
         };

crates/ide_db/src/erl_ast.rs

@@ -81,6 +81,7 @@ impl AstLoader for crate::RootDatabase {
         let req = ParseRequest {
             options,
             override_options,
+            file_id,
             path: path.clone(),
             format,
         };

erlang_service/src/elp_epp.erl

@@ -21,9 +21,9 @@
 
 %% An Erlang code preprocessor.
 
--export([open/2, open/3, open/4, close/1, format_error/1]).
+-export([open/3, open/4, open/5, close/1, format_error/1]).
 -export([scan_erl_form/1, parse_erl_form/1, macro_defs/1]).
--export([scan_file/1, scan_file/3, parse_file/1, parse_file/3, parse_file/4]).
+-export([scan_file/1, scan_file/4, parse_file/1, parse_file/4, parse_file/5]).
 -export([
     default_encoding/0,
     encoding_to_string/1,
@@ -76,6 +76,8 @@
     include_offset = 1,
     %Current file name
     name = "" :: file:name(),
+    % ELP FileId
+    file_id :: erlang_service_server:file_id(),
     %-"-, modified by -file
     name2 = "" :: file:name(),
     %Ifdef stack
@@ -110,37 +112,40 @@
 %% parse_file(FileName, IncludePath, PreDefMacros)
 %% macro_defs(Epp)
 
--spec open(Id, FileName, IncludePath) ->
+-spec open(Id, FileName, FileId, IncludePath) ->
     {'ok', Epp} | {'ok', Epp, Extra} | {'error', ErrorDescriptor}
 when
     Id :: erlang_service_server:id(),
+    FileId :: erlang_service_server:file_id(),
     FileName :: file:name(),
     IncludePath :: [DirectoryName :: file:name()],
     Epp :: epp_handle(),
     Extra :: [{'encoding', source_encoding() | 'none'}],
     ErrorDescriptor :: term().
 
-open(Id, Name, Path) ->
-    open(Id, Name, Path, []).
+open(Id, Name, FileId, Path) ->
+    open(Id, Name, FileId, Path, []).
 
--spec open(Id, FileName, IncludePath, PredefMacros) ->
+-spec open(Id, FileName, FileId, IncludePath, PredefMacros) ->
     {'ok', Epp} | {'ok', Epp, Extra} | {'error', ErrorDescriptor}
 when
     Id :: erlang_service_server:id(),
     FileName :: file:name(),
+    FileId :: erlang_service_server:file_id(),
     IncludePath :: [DirectoryName :: file:name()],
     PredefMacros :: macros(),
     Epp :: epp_handle(),
     Extra :: [{'encoding', source_encoding() | 'none'}],
     ErrorDescriptor :: term().
 
-open(Id, Name, Path, Pdm) ->
-    open(Id, [{name, Name}, {includes, Path}, {macros, Pdm}]).
+open(Id, Name, FileId, Path, Pdm) ->
+    open(Id, FileId, [{name, Name}, {includes, Path}, {macros, Pdm}]).
 
--spec open(Id, Options) ->
+-spec open(Id, FileId, Options) ->
     {'ok', Epp} | {'ok', Epp, Extra} | {'error', ErrorDescriptor}
 when
     Id :: erlang_service_server:id(),
+    FileId :: erlang_service_server:file_id(),
     Options :: [
         {'default_encoding', DefEncoding :: source_encoding()}
         | {'includes', IncludePath :: [DirectoryName :: file:name()]}
@@ -154,13 +159,13 @@ when
     Extra :: [{'encoding', source_encoding() | 'none'}],
     ErrorDescriptor :: term().
 
-open(Id, Options) ->
+open(Id, FileId, Options) ->
     case proplists:get_value(name, Options) of
         undefined ->
             erlang:error(badarg);
         Name ->
             Self = self(),
-            Epp = spawn(fun() -> server(Self, Id, Name, Options) end),
+            Epp = spawn(fun() -> server(Self, Id, Name, FileId, Options) end),
             Extra = proplists:get_bool(extra, Options),
             case epp_request(Epp) of
                 {ok, Pid, Encoding} when Extra ->
@@ -295,11 +300,12 @@ format_error(string_concat) ->
 format_error(E) ->
     file:format_error(E).
 
--spec scan_file(Id, FileName, Options) ->
+-spec scan_file(Id, FileName, FileId, Options) ->
     {'ok', [Form], Extra} | {error, OpenError}
 when
     Id :: erlang_service_server:id(),
     FileName :: file:name(),
+    FileId :: erlang_service_server:file_id(),
     Options :: [
         {'includes', IncludePath :: [DirectoryName :: file:name()]}
         | {'source_name', SourceName :: file:name()}
@@ -312,8 +318,8 @@ when
     Extra :: [{'encoding', source_encoding() | 'none'}],
     OpenError :: term().
 
-scan_file(Id, Ifile, Options) ->
-    case open(Id, [{name, Ifile}, extra | Options]) of
+scan_file(Id, Ifile, FileId, Options) ->
+    case open(Id, FileId, [{name, Ifile}, extra | Options]) of
         {ok, Epp, Extra} ->
             Forms = scan_file(Epp),
             close(Epp),
@@ -332,11 +338,12 @@ scan_file(Epp) ->
             [{eof, {Offset, Offset}}]
     end.
 
--spec parse_file(Id, FileName, IncludePath, PredefMacros) ->
+-spec parse_file(Id, FileName, FileId, IncludePath, PredefMacros) ->
     {'ok', [Form]} | {'ok', [Form], Extra} | {error, OpenError}
 when
     Id :: erlang_service_server:id(),
     FileName :: file:name(),
+    FileId :: erlang_service_server:id(),
     IncludePath :: [DirectoryName :: file:name()],
     PredefMacros :: macros(),
     Form :: elp_parse:abstract_form() | {'error', ErrorInfo} | {'eof', Location},
@@ -345,14 +352,15 @@ when
     ErrorInfo :: elp_scan:error_info() | elp_parse:error_info(),
     OpenError :: term().
 
-parse_file(Id, Ifile, Path, Predefs) ->
-    parse_file(Id, Ifile, [{includes, Path}, {macros, Predefs}]).
+parse_file(Id, Ifile, FileId, Path, Predefs) ->
+    parse_file(Id, Ifile, FileId, [{includes, Path}, {macros, Predefs}]).
 
--spec parse_file(Id, FileName, Options) ->
+-spec parse_file(Id, FileName, FileId, Options) ->
     {'ok', [Form]} | {'ok', [Form], Extra} | {error, OpenError}
 when
     Id :: erlang_service_server:id(),
     FileName :: file:name(),
+    FileId :: erlang_service_server:id(),
     Options :: [
         {'includes', IncludePath :: [DirectoryName :: file:name()]}
         | {'source_name', SourceName :: file:name()}
@@ -369,8 +377,8 @@ when
     Extra :: [{'encoding', source_encoding() | 'none'}],
     OpenError :: term().
 
-parse_file(Id,  Ifile, Options) ->
-    case open(Id, [{name, Ifile} | Options]) of
+parse_file(Id,  Ifile, FileId, Options) ->
+    case open(Id, FileId, [{name, Ifile} | Options]) of
         {ok, Epp} ->
             Forms = parse_file(Epp),
             close(Epp),
@@ -674,9 +682,9 @@ restore_typed_record_fields([
 restore_typed_record_fields([Form | Forms]) ->
     [Form | restore_typed_record_fields(Forms)].
 
-server(Pid, Id, Name, Options) ->
+server(Pid, Id, Name, FileId, Options) ->
     process_flag(trap_exit, true),
-    St = #epp{ request_id = Id},
+    St = #epp{ request_id = Id, file_id = FileId},
     case proplists:get_value(fd, Options) of
         undefined ->
             case file:open(Name, [read]) of
@@ -686,7 +694,7 @@ server(Pid, Id, Name, Options) ->
                     epp_reply(Pid, {error, E})
             end;
         Fd ->
-            init_server(Pid, Name, Options, St#epp{file = Fd, pre_opened = true})
+            init_server(Pid, Name, Options, St#epp{file = Fd, file_id = FileId, pre_opened = true})
     end.
 
 init_server(Pid, FileName, Options0, St0) ->
@@ -990,7 +998,7 @@ scan_toks([{'-', _Lh}, {atom, _Le, endif} = Endif | Toks], From, St) ->
 scan_toks([{'-', _Lh}, {atom, _Lf, file} = FileToken | Toks0], From, St) ->
     try expand_macros(Toks0, St) of
         Toks1 when is_list(Toks1) ->
-            scan_file(Toks1, FileToken, From, St)
+            scan_file_toks(Toks1, FileToken, From, St)
     catch
         {error, ErrL, What} ->
             epp_reply(From, {error, {ErrL, elp_epp, What}}),
@@ -1656,11 +1664,11 @@ scan_endif(Toks, Endif, From, St) ->
     epp_reply(From, {error, {loc(T), elp_epp, {bad, endif}}}),
     wait_req_scan(St).
 
-%% scan_file(Tokens, FileToken, From, EppState)
+%% scan_file_toks(Tokens, FileToken, From, EppState)
 %%  Set the current file and line to the given file and line.
 %%  Note that the line of the attribute itself is kept.
 
-scan_file(Tokens0, Tf, From, St) ->
+scan_file_toks(Tokens0, Tf, From, St) ->
     Tokens = coalesce_strings(Tokens0),
     scan_file1(Tokens, Tf, From, St).
 

erlang_service/src/elp_escript.erl

@@ -28,10 +28,11 @@
 
 -module(elp_escript).
 
--export([extract/2]).
+-export([extract/3]).
 
 -record(state, {
     file :: file:filename(),
+    file_id :: erlang_service_server:file_id(),
     module :: module(),
     forms_or_bin,
     exports_main :: boolean()
@@ -43,24 +44,25 @@
 -record(sections, {shebang :: shebang() | 'undefined'}).
 -type sections() :: #sections{}.
 
--spec extract(erlang_service_server:id(), file:filename()) -> any().
-extract(Id, File) ->
+-spec extract(erlang_service_server:id(), file:filename(), erlang_service_server:file_id()) -> any().
+extract(Id, File, FileId) ->
     {HeaderSz, Fd, _Sections} = parse_header(File),
-    Forms = do_parse_file(Id, File, Fd, HeaderSz),
+    Forms = do_parse_file(Id, File, FileId, Fd, HeaderSz),
     ok = file:close(Fd),
     Forms.
 
--spec do_parse_file(erlang_service_server:id(), any(), any(), any()) ->
+-spec do_parse_file(erlang_service_server:id(), any(), erlang_service_server:file_id(), any(), any()) ->
     [any()].
-do_parse_file(Id, File, Fd, HeaderSz) ->
-    S = initial_state(File),
+do_parse_file(Id, File, FileId, Fd, HeaderSz) ->
+    S = initial_state(File, FileId),
     #state{forms_or_bin = FormsOrBin} = parse_source(Id, S, File, Fd, HeaderSz),
     FormsOrBin.
 
--spec initial_state(_) -> state().
-initial_state(File) ->
+-spec initial_state(_,_) -> state().
+initial_state(File, FileId) ->
     #state{
         file = File,
+        file_id = FileId,
         exports_main = false
     }.
 
@@ -135,7 +137,7 @@ parse_source(Id, S, File, Fd, HeaderSz) ->
     _ = io:get_line(Fd, ''),
     Encoding = elp_epp:set_encoding(Fd),
     {ok, _} = file:position(Fd, HeaderSz),
-    {ok, Epp} = epp_open(Id, File, Fd, HeaderSz, IncludePath, PreDefMacros),
+    {ok, Epp} = epp_open(Id, File, S#state.file_id, Fd, HeaderSz, IncludePath, PreDefMacros),
     _ = [io:setopts(Fd, [{encoding, Encoding}]) || Encoding =/= none],
     {ok, FileForm} = elp_epp:parse_erl_form(Epp),
     OptModRes = elp_epp:parse_erl_form(Epp),
@@ -165,9 +167,9 @@ parse_source(Id, S, File, Fd, HeaderSz) ->
     ok = file:close(Fd),
     check_source(S2).
 
--spec epp_open(erlang_service_server:id(), _, _, pos_integer(), _, _) -> {ok, term()}.
-epp_open(Id, File, Fd, HeaderSz, IncludePath, PreDefMacros) ->
-    elp_epp:open(Id, [
+-spec epp_open(erlang_service_server:id(), _, erlang_service_server:file_id(), _, pos_integer(), _, _) -> {ok, term()}.
+epp_open(Id, File, FileId, Fd, HeaderSz, IncludePath, PreDefMacros) ->
+    elp_epp:open(Id, FileId, [
         {fd, Fd},
         {name, File},
         {offset, HeaderSz},

erlang_service/src/erlang_service_lint.erl

@@ -9,7 +9,7 @@
 
 -export([run/2]).
 
-run(Id, [FileName, Options0, OverrideOptions, PostProcess, Deterministic]) ->
+run(Id, [FileName, FileId, Options0, OverrideOptions, PostProcess, Deterministic]) ->
     Options1 =
         case Deterministic of
             true ->
@@ -30,7 +30,7 @@ run(Id, [FileName, Options0, OverrideOptions, PostProcess, Deterministic]) ->
             _ ->
                 Options2
         end,
-    case extract_forms(Id, FileName, Options3) of
+    case extract_forms(Id, FileName, FileId, Options3) of
         {ok, Forms0} ->
             Transforms0 = proplists:get_value(parse_transforms, Options3, []),
             {Transforms, Forms1} = collect_parse_transforms(Forms0, [], Transforms0),
@@ -242,14 +242,14 @@ inclusion_range(Forms, Path) ->
             {1, 1}
     end.
 
-extract_forms(Id, FileName, Options) ->
+extract_forms(Id, FileName, FileId, Options) ->
     case filename:extension(FileName) of
         ".erl" ->
-            elp_epp:parse_file(Id, FileName, Options);
+            elp_epp:parse_file(Id, FileName, FileId, Options);
         ".hrl" ->
-            elp_epp:parse_file(Id, FileName, Options);
+            elp_epp:parse_file(Id, FileName, FileId, Options);
         ".escript" ->
-            Forms = elp_escript:extract(Id, FileName),
+            Forms = elp_escript:extract(Id, FileName, FileId),
             {ok, Forms};
         _Ext ->
             {error, "Skipping diagnostics due to extension"}

erlang_service/src/erlang_service_server.erl

@@ -32,7 +32,7 @@
 ]).
 
 %% API
--export_type([id/0]).
+-export_type([id/0, file_id/0]).
 
 %%==============================================================================
 %% Includes
@@ -52,6 +52,7 @@
 -type result() :: {result, id(), [segment()]}.
 -type exception() :: {exception, id(), any()}.
 -type id() :: integer().
+-type file_id() :: integer().
 -type segment() :: {string(), binary()}.
 
 %%==============================================================================