Merge branch 'lukas/kernel/fix-latin1-shell-issues-maint-26/OTP-18675…

…/OTP-18676' into maint

* lukas/kernel/fix-latin1-shell-issues-maint-26/OTP-18675/OTP-18676:
  io: Add types io:standard_io/standard_error and document them
  shell: Cleanup latin1 handling in group
  shell: Update docs to describe using bytewise encoding
  kernel: Update prim_tty reader to handle blocking reads

erts/doc/src/erlang.xml

@@ -2915,8 +2915,23 @@ uncompiled code with the same arity are mapped to the same list by
           groups have a <em>group leader</em>. All I/O from the group
           is channeled to the group leader. When a new process is
           spawned, it gets the same group leader as the spawning
-          process. Initially, at system startup, <c>init</c> is both
-          its own group leader and the group leader of all processes.</p>
+          process.</p>
+        <p>Initially, at system startup, <c>init</c> is both
+          its own group leader and the group leader of all processes.
+          During the boot of a system the group leader for processes
+          will be changed depending on the need of the system. Some examples
+          where this is done are:</p>
+        <list type="bulleted">
+          <item>When an application is started, the top supervisor of that
+            application will have its group leader set to the application
+            master. See <seemfa marker="kernel:application#start/2"><c>
+            application:start/2</c></seemfa> for more details.</item>
+          <item>When running tests, both <seeapp marker="common_test:index"><c>common_test</c></seeapp> and
+            <seeerl marker="eunit:eunit"><c>eunit</c></seeerl> set the
+            group leader in order to capture any I/O from the testcase.</item>
+          <item>The <seeerl marker="stdlib:shell">interactive shell</seeerl>
+            sets the group leader to intercept I/O.</item>
+        </list>
       </desc>
     </func>
 

erts/doc/src/escript_cmd.xml

@@ -119,10 +119,8 @@ $ <input>escript factorial 5</input></pre>
             I/O-server, however, must be set explicitly as follows:</p>
           <code>
 io:setopts([{encoding, latin1}])</code>
-          <p>The default encoding of the I/O-server for <c>standard_io</c>
-            is <c>unicode</c> if its supported, as the script runs in a
-            non-interactive terminal.
-            (see section
+          <p>The default encoding of the I/O-server for <seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>
+            is <c>unicode</c> if its supported. (see section
             <seeguide marker="stdlib:unicode_usage#unicode_options_summary">
             Summary of Options</seeguide>) in the STDLIB User's Guide.</p>
         </note>

lib/common_test/doc/src/run_test_chapter.xml

@@ -1169,7 +1169,7 @@
 	<item>A confirmation when the test run is complete.</item>
 	<item>Some special information, such as error reports, progress
 	  reports, and printouts written with <c>erlang:display/1</c>, or <c>io:format/3</c>
-	  specifically addressed to a receiver other than <c>standard_io</c>
+	  specifically addressed to a receiver other than <seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>
 	  (for example, the default group leader process <c>user</c>).</item>
       </list>
 

lib/erl_docgen/priv/bin/validate_links.escript

@@ -256,13 +256,18 @@ validate_link(Filename, "seemfa", Line, Link, CachedFiles) ->
     end;
 validate_link(Filename, LinkType = "seetype", Line, Link, CachedFiles) ->
     {App,Mod,Type} = ParsedLink = parse_link(Filename, maps:get(m2a,CachedFiles), Link),
-    Types = maps:get(datatypes,maps:get({App,Mod},CachedFiles)),
-    case lists:member(Type, Types) of
-        false ->
+    case maps:find({App,Mod},CachedFiles) of
+        error ->
             fail(Line, "Could not find documentation for ~s when "
                  "resolving link",[App ++ ":" ++ Mod ++ "#" ++ Type]);
-        _ ->
-            validate_type(Line,LinkType,read_link(Line, ParsedLink, CachedFiles))
+        {ok, AppData} ->
+            case lists:member(Type, maps:get(datatypes,AppData)) of
+                false ->
+                    fail(Line, "Could not find documentation for ~s when "
+                         "resolving link",[App ++ ":" ++ Mod ++ "#" ++ Type]);
+                _ ->
+                    validate_type(Line,LinkType,read_link(Line, ParsedLink, CachedFiles))
+            end
     end;
 validate_link({"jinterface","jinterface_users_guide"},"seefile",_, _, _) ->
     %% Skip links to java documentation

lib/kernel/doc/src/file.xml

@@ -1374,8 +1374,8 @@ f.txt:  {person, "kalle", 25}.
 	  read from the file, while the position in the file can be moved much more than
 	  this number when reading a Unicode file.</p>
 	  <p>Also, if <c>encoding</c> is set to something else than <c>latin1</c>,
-	  the <c>read/3</c> call fails if the data contains characters larger than 255,
-	  which is why module <seeerl marker="stdlib:io"><c>io(3)</c></seeerl>
+	  the <c>read/2</c> call fails if the data contains characters larger than 255,
+	  which is why <seemfa marker="stdlib:io#get_chars/3"><c>io:get_chars/3</c></seemfa>
 	  is to be preferred when reading such a file.</p>
 	  <p>The function returns:</p>
         <taglist>
@@ -1628,7 +1628,7 @@ f.txt:  {person, "kalle", 25}.
 	  raw line-oriented reading.</p>
 	  <p>If <c>encoding</c> is set to something else than <c>latin1</c>, the
 	  <c>read_line/1</c> call fails if the data contains characters larger than 255,
-	  why module <seeerl marker="stdlib:io"><c>io(3)</c></seeerl> is to be
+	  why <seemfa marker="stdlib:io#get_line/2"><c>io:get_line/2</c></seemfa> is to be
 	  preferred when reading such a file.</p>
 	  <p>The function returns:</p>
         <taglist>
@@ -1978,7 +1978,10 @@ f.txt:  {person, "kalle", 25}.
 	  <p>If the file is opened with <c>encoding</c> set to something else than
 	  <c>latin1</c>, each byte written can result in many bytes being written to
 	  the file, as the byte range 0..255 can represent anything between one and
-	  four bytes depending on value and UTF encoding type.</p>
+	  four bytes depending on value and UTF encoding type. If you want to write
+          <seetype marker="stdlib:unicode#chardata"><c>unicode:chardata()</c></seetype>
+          to the <c><anno>IoDevice</anno></c> you should use <seemfa marker="stdlib:io#put_chars/2">
+          <c>io:put_chars/2</c></seemfa> instead.</p>
         <p>Typical error reasons:</p>
         <taglist>
           <tag><c>ebadf</c></tag>

lib/kernel/doc/src/kernel_app.xml

@@ -614,13 +614,13 @@ MaxT = NetTickTime + NetTickTime / NetTickIntensity</code>
 	</p>
 
       </item>
-    <tag><marker id="standard_io_encoding"/><c>standard_io_encoding = Encoding</c></tag>
+      <tag><marker id="standard_io_encoding"/><c>standard_io_encoding = Encoding</c></tag>
       <item>
         <p>Set whether bytes sent or received via standard_io should be interpreted as unicode or latin1.
-        By default input and output is interpreted as Unicode if it is supported on the host. With this flag
-        you may configure the encoding on startup.</p>
+          By default input and output is interpreted as Unicode if it is supported on the host. With this flag
+          you may configure the encoding on startup.</p>
         <p>This works similarly to <seemfa marker="stdlib:io#setopts/2"><c>io:setopts(standard_io, {encoding, Encoding})</c></seemfa>
-        but is applied before any bytes on standard_io may have been read.</p>
+          but is applied before any bytes on standard_io may have been read.</p>
         <p>Encoding is one of:</p>
         <taglist>
           <tag><c>unicode</c></tag>
@@ -631,8 +631,10 @@ MaxT = NetTickTime + NetTickTime / NetTickIntensity</code>
           <item><p>Anything other than unicode or latin1 will be ignored and the system will
             configure the encoding by itself, typically unicode on modern systems.</p></item>
         </taglist>
+        <p>See <seeguide marker="stdlib:unicode_usage#escripts-and-non-interactive-i-o">
+        Escripts and non-interactive I/O in Unicode Usage in Erlang</seeguide> for more details.</p>
       </item>
-          </taglist>
+    </taglist>
   </section>
 
   <section>

lib/kernel/doc/src/logger_chapter.xml

@@ -829,7 +829,7 @@ logger:debug(#{got => connection_request, id => Id, state => State},
 	configuration file that configures Logger according to the
 	description.</p>
       <p>Modify the default handler to print to a file instead of
-	<c>standard_io</c>:</p>
+	<seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>:</p>
       <code>
 [{kernel,
   [{logger,

lib/kernel/doc/src/logger_std_h.xml

@@ -38,8 +38,8 @@
   <description>
     <p>This is the standard handler for Logger.
       Multiple instances of this handler can be added to
-      Logger, and each instance prints logs to <c>standard_io</c>,
-      <c>standard_error</c>, or to file.</p>
+      Logger, and each instance prints logs to <seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>,
+      <seetype marker="stdlib:io#standard_error"><c>standard_error</c></seetype>, or to file.</p>
     <p>The handler has an overload protection mechanism that keeps the handler
       process and the Kernel application alive during high loads of log 
       events. How overload protection works, and how to configure it, is
@@ -55,12 +55,14 @@
        is stored in a sub map with the key <c>config</c>, and can contain the
        following parameters:</p>
     <taglist>
-      <tag><marker id="type"/><c>type = standard_io | standard_error | file | {device, io:device()}</c></tag>
+      <tag><marker id="type"/><c>type = </c><seetype marker="stdlib:io#standard_io"><c>io:standard_io()</c></seetype><c>
+        | </c><seetype marker="stdlib:io#standard_error"><c>io:standard_error()</c></seetype><c> | file
+        | {device, </c><seetype marker="stdlib:io#device"><c>io:device()</c></seetype><c>}</c></tag>
       <item>
 	<p>Specifies the log destination.</p>
 	<p>The value is set when the handler is added, and it cannot
 	  be changed in runtime.</p>
-	<p>Defaults to <c>standard_io</c>, unless
+	<p>Defaults to <seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>, unless
 	  parameter <seeerl marker="#file"><c>file</c></seeerl> is
 	  given, in which case it defaults to <c>file</c>.</p>
       </item>
@@ -182,7 +184,7 @@ logger:add_handler(my_standard_h, logger_std_h,
                                  filesync_repeat_interval => 1000}}).
     </code>
     <p>To set the default handler, that starts initially with
-    the Kernel application, to log to file instead of <c>standard_io</c>,
+    the Kernel application, to log to file instead of <seetype marker="stdlib:io#standard_io"><c>standard_io</c></seetype>,
     change the Kernel default logger configuration. Example:</p>
     <code type="none">
 erl -kernel logger '[{handler,default,logger_std_h,

lib/kernel/src/file.erl

@@ -582,7 +582,7 @@ allocate(#file_descriptor{module = Module} = Handle, Offset, Length) ->
     Module:allocate(Handle, Offset, Length).
 
 -spec read(IoDevice, Number) -> {ok, Data} | eof | {error, Reason} when
-      IoDevice :: io_device() | atom(),
+      IoDevice :: io_device() | io:device(),
       Number :: non_neg_integer(),
       Data :: string() | binary(),
       Reason :: posix()
@@ -604,7 +604,7 @@ read(_, _) ->
     {error, badarg}.
 
 -spec read_line(IoDevice) -> {ok, Data} | eof | {error, Reason} when
-      IoDevice :: io_device() | atom(),
+      IoDevice :: io_device() | io:device(),
       Data :: string() | binary(),
       Reason :: posix()
               | badarg
@@ -668,7 +668,7 @@ pread(_, _, _) ->
     {error, badarg}.
 
 -spec write(IoDevice, Bytes) -> ok | {error, Reason} when
-      IoDevice :: io_device() | atom(),
+      IoDevice :: io_device() | io:device(),
       Bytes :: iodata(),
       Reason :: posix() | badarg | terminated.
 

lib/kernel/src/group.erl

@@ -47,7 +47,6 @@ server(Ancestors, Drv, Shell, Options) ->
     put(line_buffer, proplists:get_value(line_buffer, Options, group_history:load())),
     put(read_mode, list),
     put(user_drv, Drv),
-    put(unicode_state, true),
     ExpandFun = normalize_expand_fun(Options, fun edlin_expand:expand/2),
     put(expand_fun, ExpandFun),
     put(echo, proplists:get_value(echo, Options, true)),
@@ -217,8 +216,7 @@ io_request(Req, From, ReplyAs, Drv, Shell, Buf0) ->
 
 
 %% Put_chars, unicode is the normal message, characters are always in
-%%standard unicode
-%% format.
+%% standard unicode format.
 %% You might be tempted to send binaries unchecked, but the driver
 %% expects unicode, so that is what we should send...
 %% io_request({put_chars,unicode,Binary}, Drv, Buf) when is_binary(Binary) ->
@@ -250,27 +248,17 @@ io_request({put_chars,unicode,M,F,As}, Drv, _Shell, From, Buf) ->
 	    end
     end;
 io_request({put_chars,latin1,Binary}, Drv, _Shell, From, Buf) when is_binary(Binary) ->
-    IsUnicode = get(unicode_state),
-    if IsUnicode ->
-            send_drv(Drv,
-                     {put_chars_sync, unicode,
-                      unicode:characters_to_binary(Binary,latin1),
-                      From});
-        true ->
-            send_drv(Drv, {put_chars_sync, latin1, Binary, From})
-    end,
+    send_drv(Drv, {put_chars_sync, unicode,
+                   unicode:characters_to_binary(Binary,latin1),
+                   From}),
     {noreply,Buf};
 io_request({put_chars,latin1,Chars}, Drv, _Shell, From, Buf) ->
-    IsUnicode = get(unicode_state),
-    if IsUnicode ->
-            case catch unicode:characters_to_binary(Chars,latin1) of
-                Binary when is_binary(Binary) ->
-                    send_drv(Drv, {put_chars_sync, unicode, Binary, From}),
-                    {noreply,Buf};
-                _ ->
-                    {error,{error,{put_chars,latin1,Chars}},Buf}
-            end;
-        true -> send_drv(Drv, {put_chars_sync, latin1, Chars, From})
+    case catch unicode:characters_to_binary(Chars,latin1) of
+        Binary when is_binary(Binary) ->
+            send_drv(Drv, {put_chars_sync, unicode, Binary, From}),
+            {noreply,Buf};
+        _ ->
+            {error,{error,{put_chars,latin1,Chars}},Buf}
     end;
 io_request({put_chars,latin1,M,F,As}, Drv, _Shell, From, Buf) ->
     case catch apply(M, F, As) of
@@ -401,28 +389,29 @@ setopts(Opts0,Drv,Buf) ->
     end.
 check_valid_opts([]) ->
     true;
-check_valid_opts([{binary,_}|T]) ->
+check_valid_opts([{binary,Flag}|T]) when is_boolean(Flag) ->
     check_valid_opts(T);
-check_valid_opts([{encoding,Valid}|T]) when Valid =:= unicode; Valid =:= utf8; Valid =:= latin1 ->
+check_valid_opts([{encoding,Valid}|T]) when Valid =:= unicode;
+                                            Valid =:= utf8;
+                                            Valid =:= latin1 ->
     check_valid_opts(T);
-check_valid_opts([{echo,_}|T]) ->
+check_valid_opts([{echo,Flag}|T]) when is_boolean(Flag) ->
     check_valid_opts(T);
-check_valid_opts([{expand_fun,_}|T]) ->
+check_valid_opts([{expand_fun,Fun}|T]) when is_function(Fun, 1);
+                                            is_function(Fun, 2) ->
     check_valid_opts(T);
 check_valid_opts(_) ->
     false.
 
 do_setopts(Opts, Drv, Buf) ->
     put(expand_fun, normalize_expand_fun(Opts, get(expand_fun))),
     put(echo, proplists:get_value(echo, Opts, get(echo))),
-    case proplists:get_value(encoding,Opts) of
+    case proplists:get_value(encoding, Opts) of
 	Valid when Valid =:= unicode; Valid =:= utf8 ->
-           set_unicode_state(Drv,true),
-           put(unicode_state, true);
+           set_unicode_state(Drv,true);
 	latin1 ->
-           set_unicode_state(Drv,false),
-           put(unicode_state, false);
-	_ ->
+           set_unicode_state(Drv,false);
+	undefined ->
 	    ok
     end,
     case proplists:get_value(binary, Opts, case get(read_mode) of
@@ -434,8 +423,6 @@ do_setopts(Opts, Drv, Buf) ->
 	    {ok,ok,Buf};
 	false ->
 	    put(read_mode, list),
-	    {ok,ok,Buf};
-	_ ->
 	    {ok,ok,Buf}
     end.