Connect to jupyter kernels

DEVELOPER-GUIDE.md

@@ -198,3 +198,49 @@ else
    - If you get an error about access being denied, ensure that all other Notebooks are closed and then restart the kernel again as in step 5
 
 8. Now, use the kernel as you normally would. You should see your local changes being used by the extension.
+
+
+### Setup full suite of tests to run
+
+Some tests require additional setup or will be skipped. `JupyterKernel` tests for e.g. are setup to have the same test run against a Jupyter server, directly against the Jupyter kernel over ZMQ and with a simulation of the messages. Jupyter server and Jupyter kernel tests require the following setup or will be skipped while the tests with simulation can be run without additional steps. 
+
+### Run tests with a local Jupyter Server
+
+1. Install [Jupyter server](https://docs.jupyter.org/en/latest/install.html) or [Anaconda](https://www.anaconda.com/products/distribution)
+2. [Install R kernel](https://docs.anaconda.com/anaconda/user-guide/tasks/using-r-language/) for R tests by calling the following in Anaconda Prompt (Windows) or the terminal (Mac/Linux)
+```
+conda install -c r r-irkernel
+```
+3. Start the server locally as mentioned [here](https://docs.jupyter.org/en/latest/running.html). You can use any random string or guid for your_token value.
+```
+jupyter notebook --no-browser --NotebookApp.token=<your_token> --port=8888
+```
+4. Set an environment variable `TEST_DOTNET_JUPYTER_HTTP_CONN` pointing to the server and the token you are using for the Jupyter server as 
+```
+--url http://localhost:8888 --token <your_token>
+```
+5. The tests will now use the environment variable to connect to your server. 
+
+### Run tests with a Jupyter Kernel over ZMQ
+
+1. Install [Anaconda](https://www.anaconda.com/products/distribution)
+2. [Install R kernel](https://docs.anaconda.com/anaconda/user-guide/tasks/using-r-language/) for R tests by calling the following in Anaconda Prompt (Windows) or the terminal (Mac/Linux) 
+```
+conda install -c r r-irkernel
+```
+3. Start Anaconda Bash prompt and create an environment variable `TEST_DOTNET_JUPYTER_ZMQ_CONN` and set to `true` and reactivate your conda environment
+```bash
+conda env config vars set TEST_DOTNET_JUPYTER_ZMQ_CONN=true 
+conda activate base
+```
+3. Restart `dotnet-interactive.sln` from the Anaconda Bash prompt. 
+4. The tests will now use the environment variable to connect to your server. 
+
+### Run tests directly against the language handler scripts
+
+These tests can be run directly against the language handler scripts. This is useful for when making changes on the scripts sent to the jupyter kernel without needing a full integration.
+
+1. Python tests can be run directly in the Anaconda Prompt with IPython by calling `src\Microsoft.DotNet.Interactive.Jupyter.Tests\LanguageHandlerTests\run_python_tests.bat`
+2. R tests can be run directly in the Anaconda Prompt with RScript by calling `src\Microsoft.DotNet.Interactive.Jupyter.Tests\LanguageHandlerTests\run_r_tests.bat`
+3. Both Python and R tests can be run together in the Anaconda Prompt by calling `src\Microsoft.DotNet.Interactive.Jupyter.Tests\LanguageHandlerTests\run_tests.bat`
+

NOTICE.txt

@@ -25683,3 +25683,39 @@ For more information, please refer to <http://unlicense.org/>
 
 ---------------------------------------------------------
 
+Some parts of Jupyter messaging protocol handling and Jupyter Kernel creation are inspired from the nteract OSS project.
+
+nteract
+https://github.com/nteract/nteract
+BSD-3-Clause license
+
+Copyright (c) 2016, nteract contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of nteract nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+---------------------------------------------------------
+

src/Microsoft.DotNet.Interactive.ApiCompatibility.Tests/ApiCompatibilityTests.jupyter_api_is_not_changed.approved.txt

@@ -15,6 +15,12 @@ Microsoft.DotNet.Interactive.Jupyter
     public System.String SignatureScheme { get; set;}
     public System.Int32 StdinPort { get; set;}
     public System.String Transport { get; set;}
+  public class ConnectJupyterKernelCommand : Microsoft.DotNet.Interactive.Connection.ConnectKernelCommand, System.Collections.Generic.IEnumerable<System.CommandLine.Symbol>, System.Collections.IEnumerable, System.CommandLine.Completions.ICompletionSource
+    .ctor()
+    public System.CommandLine.Option<System.String> InitScript { get;}
+    public System.CommandLine.Option<System.String> KernelSpecName { get;}
+    public ConnectJupyterKernelCommand AddConnectionOptions(Microsoft.DotNet.Interactive.Jupyter.Connection.IJupyterKernelConnectionOptions connectionOptions)
+    public System.Threading.Tasks.Task<Microsoft.DotNet.Interactive.Kernel> ConnectKernelAsync(Microsoft.DotNet.Interactive.KernelInvocationContext context, System.CommandLine.Invocation.InvocationContext commandLineContext)
   public static class Constants
   public class ExecuteRequestHandler : RequestHandlerBase<Microsoft.DotNet.Interactive.Jupyter.Protocol.ExecuteRequest>, System.IDisposable
     .ctor(Microsoft.DotNet.Interactive.Kernel kernel, System.Reactive.Concurrency.IScheduler scheduler = null)
@@ -24,7 +30,11 @@ Microsoft.DotNet.Interactive.Jupyter
     .ctor(ConnectionInformation connectionInformation)
     public System.Threading.Tasks.Task StartAsync(System.Threading.CancellationToken cancellationToken)
     public System.Threading.Tasks.Task StopAsync(System.Threading.CancellationToken cancellationToken)
-  public abstract class IJupyterMessageSender
+  public abstract class IJupyterKernelSpecModule
+    public System.IO.DirectoryInfo GetDefaultKernelSpecDirectory()
+    public System.Threading.Tasks.Task<Microsoft.DotNet.Interactive.Utility.CommandLineResult> InstallKernel(System.IO.DirectoryInfo sourceDirectory)
+    public System.Threading.Tasks.Task<System.Collections.Generic.IReadOnlyDictionary<System.String,KernelSpec>> ListKernels()
+  public abstract class IJupyterMessageResponseSender
     public System.Void Send(Microsoft.DotNet.Interactive.Jupyter.Protocol.PubSubMessage message)
     public System.Void Send(Microsoft.DotNet.Interactive.Jupyter.Protocol.ReplyMessage message)
     public System.String Send(Microsoft.DotNet.Interactive.Jupyter.Protocol.InputRequest message)
@@ -39,9 +49,24 @@ Microsoft.DotNet.Interactive.Jupyter
   public class JupyterClientKernelExtension, Microsoft.DotNet.Interactive.IKernelExtension
     .ctor()
     public System.Threading.Tasks.Task OnLoadAsync(Microsoft.DotNet.Interactive.Kernel kernel)
+  public class JupyterHttpKernelConnectionOptions, Microsoft.DotNet.Interactive.Jupyter.Connection.IJupyterKernelConnectionOptions
+    .ctor()
+    public System.CommandLine.Option<System.String> TargetUrl { get;}
+    public System.CommandLine.Option<System.String> Token { get;}
+    public Microsoft.DotNet.Interactive.Jupyter.Connection.IJupyterConnection GetConnection(System.CommandLine.Parsing.ParseResult connectionOptionsParseResult)
+    public System.Collections.Generic.IReadOnlyCollection<System.CommandLine.Option> GetOptions()
+  public class JupyterKernelSpecModule, IJupyterKernelSpecModule
+    .ctor()
+    public System.IO.DirectoryInfo GetDefaultKernelSpecDirectory()
+    public System.Threading.Tasks.Task<Microsoft.DotNet.Interactive.Utility.CommandLineResult> InstallKernel(System.IO.DirectoryInfo sourceDirectory)
+    public System.Threading.Tasks.Task<System.Collections.Generic.IReadOnlyDictionary<System.String,KernelSpec>> ListKernels()
+  public class JupyterLocalKernelConnectionOptions, Microsoft.DotNet.Interactive.Jupyter.Connection.IJupyterKernelConnectionOptions
+    .ctor()
+    public Microsoft.DotNet.Interactive.Jupyter.Connection.IJupyterConnection GetConnection(System.CommandLine.Parsing.ParseResult connectionOptionsParseResult)
+    public System.Collections.Generic.IReadOnlyCollection<System.CommandLine.Option> GetOptions()
   public class JupyterRequestContext
-    .ctor(IJupyterMessageSender jupyterMessageSender, Microsoft.DotNet.Interactive.Jupyter.Messaging.Message request)
-    public IJupyterMessageSender JupyterMessageSender { get;}
+    .ctor(IJupyterMessageResponseSender jupyterMessageSender, Microsoft.DotNet.Interactive.Jupyter.Messaging.Message request)
+    public IJupyterMessageResponseSender JupyterMessageSender { get;}
     public Microsoft.DotNet.Interactive.Jupyter.Messaging.Message JupyterRequestMessageEnvelope { get;}
     public System.String Token { get;}
     public System.Void Complete()
@@ -62,6 +87,15 @@ Microsoft.DotNet.Interactive.Jupyter
   public static class KernelFormattingExtensions
     public static Microsoft.DotNet.Interactive.CSharp.CSharpKernel UseMathAndLaTeX()
     public static Microsoft.DotNet.Interactive.FSharp.FSharpKernel UseMathAndLaTeX()
+  public class KernelSpec
+    .ctor()
+    public System.Collections.Generic.IReadOnlyList<System.String> CommandArguments { get; set;}
+    public System.String DisplayName { get; set;}
+    public System.Collections.Generic.IReadOnlyDictionary<System.String,System.String> EnvironmentVariables { get; set;}
+    public System.String InterruptMode { get; set;}
+    public System.String Language { get; set;}
+    public System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> Metadata { get; set;}
+    public System.String Name { get; set;}
   public abstract class RequestHandlerBase<T>, System.IDisposable
     public System.Void Dispose()
     protected Microsoft.DotNet.Interactive.Kernel get_Kernel()
@@ -83,6 +117,18 @@ Microsoft.DotNet.Interactive.Jupyter
   public static class TopLevelMethods
     public static System.String input(System.String prompt = )
     public static Microsoft.DotNet.Interactive.PasswordString password(System.String prompt = )
+Microsoft.DotNet.Interactive.Jupyter.Connection
+  public abstract class IJupyterConnection, System.IDisposable
+    public System.Threading.Tasks.Task<IJupyterKernelConnection> CreateKernelConnectionAsync(System.String kernelSpecName)
+    public System.Threading.Tasks.Task<System.Collections.Generic.IEnumerable<Microsoft.DotNet.Interactive.Jupyter.KernelSpec>> GetKernelSpecsAsync()
+  public abstract class IJupyterKernelConnection, System.IDisposable
+    public Microsoft.DotNet.Interactive.Jupyter.Messaging.IMessageReceiver Receiver { get;}
+    public Microsoft.DotNet.Interactive.Jupyter.Messaging.IMessageSender Sender { get;}
+    public System.Uri Uri { get;}
+    public System.Threading.Tasks.Task StartAsync()
+  public abstract class IJupyterKernelConnectionOptions
+    public IJupyterConnection GetConnection(System.CommandLine.Parsing.ParseResult connectionOptionsParseResult)
+    public System.Collections.Generic.IReadOnlyCollection<System.CommandLine.Option> GetOptions()
 Microsoft.DotNet.Interactive.Jupyter.Formatting
   public class LaTeXString
     public static LaTeXString op_Implicit(System.String source)
@@ -103,15 +149,19 @@ Microsoft.DotNet.Interactive.Jupyter.Messaging
     public System.String Session { get;}
     public System.String Username { get;}
     public System.String Version { get;}
+  public abstract class IMessageReceiver
+    public System.IObservable<Message> Messages { get;}
+  public abstract class IMessageSender
+    public System.Threading.Tasks.Task SendAsync(Message message)
   public static class JsonElementExtensions
     public static System.Object[] ToArray()
     public static System.Collections.Generic.IDictionary<System.String,System.Object> ToDictionary()
     public static System.Object ToObject()
     public static System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> ToReadOnlyDictionary()
   public class Message
-    public static Message Create<T>(T content, Header parentHeader = null, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> identifiers = null, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> metaData = null, System.String signature = null, System.String channel = null)
+    public static Message Create<T>(T content, Header parentHeader = null, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> identifiers = null, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> metaData = null, System.String signature = null, System.String channel = shell)
     public static Message CreatePubSub<T>(T content, Message request, System.String kernelIdentity = null)
-    public static Message CreateReply<T>(T content, Message request, System.String channel = null)
+    public static Message CreateReply<T>(T content, Message request, System.String channel = shell)
     .ctor(Header header, Microsoft.DotNet.Interactive.Jupyter.Protocol.Message content = null, Header parentHeader = null, System.String signature = null, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> metaData = null, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> identifiers = null, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> buffers = null, System.String channel = shell)
     public System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> Buffers { get;}
     public System.String Channel { get;}
@@ -125,6 +175,12 @@ Microsoft.DotNet.Interactive.Jupyter.Messaging
     public static T DeserializeFromJsonString<T>(System.String source)
     public static Message DeserializeMessage(System.String signature, System.String headerJson, System.String parentHeaderJson, System.String metadataJson, System.String contentJson, System.Collections.Generic.IReadOnlyList<System.Collections.Generic.IReadOnlyList<System.Byte>> identifiers, System.Text.Json.JsonSerializerOptions options = null)
     public static System.Boolean IsEmptyJson(System.String source)
+  public static class MessageFormatter
+    public static System.Text.Json.JsonSerializerOptions SerializerOptions { get;}
+  public static class MessageObservableExtensions
+    public static System.IObservable<Microsoft.DotNet.Interactive.Jupyter.Protocol.Message> Content()
+    public static System.IObservable<Message> ResponseOf(Message parentMessage)
+    public static System.IObservable<Microsoft.DotNet.Interactive.Jupyter.Protocol.Message> TakeUntilMessageType(System.String[] messageTypes)
   public static class MetadataExtensions
     public static System.Collections.Generic.Dictionary<System.String,System.Object> DeserializeMetadataFromJsonString(System.String metadataJson)
 Microsoft.DotNet.Interactive.Jupyter.Protocol
@@ -143,7 +199,7 @@ Microsoft.DotNet.Interactive.Jupyter.Protocol
     .ctor(System.Collections.Generic.IReadOnlyDictionary<System.String,CommTarget> comms)
     public System.Collections.Generic.IReadOnlyDictionary<System.String,CommTarget> Comms { get;}
   public class CommInfoRequest : RequestMessage
-    .ctor(System.String targetName)
+    .ctor(System.String targetName = null)
     public System.String TargetName { get;}
   public class CommMsg : Message
     .ctor(System.String commId, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> data = null)
@@ -158,16 +214,23 @@ Microsoft.DotNet.Interactive.Jupyter.Protocol
     .ctor(System.String targetName)
     public System.String TargetName { get; set;}
   public class CompleteReply : ReplyMessage
-    .ctor(System.Int32 cursorStart = 0, System.Int32 cursorEnd = 0, System.Collections.Generic.IReadOnlyList<System.String> matches = null, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> metaData = null, System.String status = null)
+    .ctor(System.Int32 cursorStart = 0, System.Int32 cursorEnd = 0, System.Collections.Generic.IReadOnlyList<System.String> matches = null, System.Collections.Generic.IReadOnlyDictionary<System.String,System.Collections.Generic.IReadOnlyList<CompletionResultMetadata>> metaData = null, System.String status = null)
     public System.Int32 CursorEnd { get;}
     public System.Int32 CursorStart { get;}
     public System.Collections.Generic.IReadOnlyList<System.String> Matches { get;}
-    public System.Collections.Generic.IReadOnlyDictionary<System.String,System.Object> MetaData { get;}
+    public System.Collections.Generic.IReadOnlyDictionary<System.String,System.Collections.Generic.IReadOnlyList<CompletionResultMetadata>> MetaData { get;}
     public System.String Status { get;}
   public class CompleteRequest : RequestMessage
     .ctor(System.String code, System.Int32 cursorPosition = 0)
     public System.String Code { get; set;}
     public System.Int32 CursorPosition { get; set;}
+  public class CompletionResultMetadata
+    .ctor(System.Int32 start = 0, System.Int32 end = 0, System.String text = null, System.String type = null, System.String displayText = null)
+    public System.String DisplayText { get;}
+    public System.Int32 End { get;}
+    public System.Int32 Start { get;}
+    public System.String Text { get;}
+    public System.String Type { get;}
   public class CSharpLanguageInfo : LanguageInfo
     .ctor(System.String version = 11.0)
   public class DisplayData : PubSubMessage