Merge branch 'master' of https://github.com/ballerina-platform/baller…

…ina-lang into issue_39641_changes

.github/workflows/fossa_scan.yml

@@ -0,0 +1,11 @@
+name: Fossa Scan
+on:
+  workflow_dispatch:
+jobs:
+  fossa-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: fossas/fossa-action@main
+        with:
+          api-key: ${{secrets.FOSSA_APIKEY}}

.github/workflows/fossa_scan_1.2.x.yml

@@ -0,0 +1,12 @@
+name: Fossa Scan
+on:
+  workflow_dispatch:
+jobs:
+  fossa-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: fossas/fossa-action@main
+        with:
+          api-key: ${{secrets.FOSSA_APIKEY}}
+          branch: ballerina-1.2.x

.github/workflows/language_server_simulator.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: 
-        branch: ["master", "2201.2.x", "2201.3.x", "2201.4.x", "2201.4.0-stage"]
+        branch: ["master", "2201.3.x", "2201.4.x", "2201.4.2-stage", "2201.5.x", "2201.6.x", "2201.6.0-stage"]
         skipGenerators: ["", "IMPORT_STATEMENT"]
 
     steps:

.github/workflows/nightly_publish_timestamped_release.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       max-parallel: 4
       matrix:
-        branch: ['master', '2201.2.x', '2201.3.x', '2201.4.x']
+        branch: ['master', '2201.4.x', '2201.5.x', '2201.6.x']
     timeout-minutes: 240
     if: github.repository_owner == 'ballerina-platform'
     steps:

.github/workflows/publish_timestamped_release.yml

@@ -1,6 +1,9 @@
 name: (Manually) Publish Timestamped Snapshot Artifacts
 
 on:
+  push:
+    branches:
+      - query-grouping-aggregation
   workflow_dispatch:
 
 jobs:

.github/workflows/pull_request_full_build.yml

@@ -1,7 +1,9 @@
 name: CI Full Build Ubuntu
 
 on:
-  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
 
 jobs:
   build-lang:
@@ -99,6 +101,9 @@ jobs:
         env:
           packageUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
           packagePAT: ${{ secrets.GITHUB_TOKEN }}
+          CLIENT_ID: ${{ secrets.CLIENT_ID }}
+          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+          REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
 
   build-distribution:
     needs: build-lang

.github/workflows/pull_request_ubuntu_build.yml

@@ -14,6 +14,7 @@ on:
       - 2201.[0-9]+.[0-9]+-stage
       - native-build
       - revert-client-decl-master
+      - query-grouping-aggregation
 
 jobs:
   ubuntu_build:

.github/workflows/pull_request_windows_build.yml

@@ -14,6 +14,7 @@ on:
       - 2201.[0-9]+.[0-9]+-stage
       - native-build
       - revert-client-decl-master
+      - query-grouping-aggregation
 
 jobs:
   windows_build:

README.md

@@ -38,7 +38,7 @@ the source code.
 
 You can use one of the following options to try out Ballerina.
 
-* [Set up Ballerina](https://ballerina.io/learn/install-ballerina/set-up-ballerina/)
+* [Set up Ballerina](https://ballerina.io/learn/get-started/)
 * [Ballerina Playground](https://play.ballerina.io/)
 
 You can use following resources to learn Ballerina.
@@ -52,7 +52,7 @@ For instructions on downloading and installing, see [Ballerina Downloads](https:
 
 ### Installation options
 
-For more installation options, see [Installation options](https://ballerina.io/learn/install-ballerina/installation-options/).
+For more installation options, see [Installation options](https://ballerina.io/downloads/installation-options/).
 
 ### Get the VS Code extension
 

ballerina-shell/modules/shell-cli/src/main/java/io/ballerina/shell/cli/utils/IncompleteInputFinder.java

@@ -24,6 +24,7 @@
 import io.ballerina.compiler.syntax.tree.BracedExpressionNode;
 import io.ballerina.compiler.syntax.tree.BuiltinSimpleNameReferenceNode;
 import io.ballerina.compiler.syntax.tree.ClassDefinitionNode;
+import io.ballerina.compiler.syntax.tree.CollectClauseNode;
 import io.ballerina.compiler.syntax.tree.ConditionalExpressionNode;
 import io.ballerina.compiler.syntax.tree.ExplicitAnonymousFunctionExpressionNode;
 import io.ballerina.compiler.syntax.tree.ExpressionFunctionBodyNode;
@@ -50,6 +51,7 @@
 import io.ballerina.compiler.syntax.tree.SelectClauseNode;
 import io.ballerina.compiler.syntax.tree.ServiceDeclarationNode;
 import io.ballerina.compiler.syntax.tree.SimpleNameReferenceNode;
+import io.ballerina.compiler.syntax.tree.SyntaxKind;
 import io.ballerina.compiler.syntax.tree.TableConstructorExpressionNode;
 import io.ballerina.compiler.syntax.tree.TransactionStatementNode;
 import io.ballerina.compiler.syntax.tree.TypeCastExpressionNode;
@@ -225,7 +227,9 @@ public Boolean transform(MappingConstructorExpressionNode node) {
     @Override
     public Boolean transform(QueryExpressionNode node) {
         return node.queryPipeline().fromClause().fromKeyword().isMissing()
-                || node.selectClause().expression().apply(this);
+                || node.resultClause().kind() == SyntaxKind.SELECT_CLAUSE ?
+                ((SelectClauseNode) node.resultClause()).expression().apply(this) :
+                ((CollectClauseNode) node.resultClause()).expression().apply(this);
     }
 
     @Override

ballerina-shell/modules/shell-core/src/main/java/io/ballerina/shell/EvaluatorImpl.java

@@ -46,7 +46,7 @@
  *
  * @since 2.0.0
  */
-class EvaluatorImpl extends Evaluator {
+public class EvaluatorImpl extends Evaluator {
     protected EvaluatorImpl(Preprocessor preprocessor, TreeParser treeParser,
                             SnippetFactory snippetFactory, ShellSnippetsInvoker invoker) {
         super(preprocessor, treeParser, snippetFactory, invoker);

ballerina-shell/modules/shell-core/src/main/java/io/ballerina/shell/invoker/ShellSnippetsInvoker.java

@@ -26,13 +26,10 @@
 import io.ballerina.projects.Document;
 import io.ballerina.projects.DocumentId;
 import io.ballerina.projects.JBallerinaBackend;
-import io.ballerina.projects.JarLibrary;
 import io.ballerina.projects.JarResolver;
 import io.ballerina.projects.JvmTarget;
 import io.ballerina.projects.Module;
-import io.ballerina.projects.ModuleName;
 import io.ballerina.projects.PackageCompilation;
-import io.ballerina.projects.PlatformLibrary;
 import io.ballerina.projects.Project;
 import io.ballerina.projects.directory.SingleFileProject;
 import io.ballerina.projects.util.ProjectUtils;
@@ -45,7 +42,6 @@
 import io.ballerina.shell.exceptions.InvokerPanicException;
 import io.ballerina.shell.invoker.classload.context.ClassLoadContext;
 import io.ballerina.shell.snippet.Snippet;
-import io.ballerina.shell.utils.Identifier;
 import io.ballerina.shell.utils.StringUtils;
 import io.ballerina.tools.diagnostics.DiagnosticSeverity;
 
@@ -56,20 +52,15 @@
 import java.io.StringWriter;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Invoker that invokes a command to evaluate a list of snippets.
@@ -86,6 +77,8 @@
 public abstract class ShellSnippetsInvoker extends DiagnosticReporter {
     /* Constants related to execution */
     protected static final String MODULE_RUN_METHOD_NAME = "__run";
+
+    protected static final String MODULE_STATEMENT_METHOD_NAME = "__stmt";
     private static final String MODULE_INIT_CLASS_NAME = "$_init";
     private static final String CONFIGURE_INIT_CLASS_NAME = "$configurationMapper";
     private static final String MODULE_INIT_METHOD_NAME = "$moduleInit";
@@ -109,10 +102,6 @@ public abstract class ShellSnippetsInvoker extends DiagnosticReporter {
      */
     private File bufferFile;
 
-    private ClassLoader classLoader = null;
-
-    private ClassLoader importClassLoader = null;
-
     protected ShellSnippetsInvoker() {
         this.scheduler = new Scheduler(false);
     }
@@ -356,13 +345,11 @@ private boolean containsModuleNotFoundError(PackageCompilation compilation) {
      * @param templateName Template to evaluate.
      * @throws InvokerException If execution/compilation failed.
      */
-    protected void executeProject(ClassLoadContext context, String templateName,
-                                  Map<Identifier, Set<Identifier>> newImports)
-            throws InvokerException {
+    protected void executeProject(ClassLoadContext context, String templateName) throws InvokerException {
         Project project = getProject(context, templateName);
         PackageCompilation compilation = compile(project);
         JBallerinaBackend jBallerinaBackend = JBallerinaBackend.from(compilation, JvmTarget.JAVA_11);
-        executeProject(jBallerinaBackend, project, newImports);
+        executeProject(jBallerinaBackend);
     }
 
     /**
@@ -373,9 +360,7 @@ protected void executeProject(ClassLoadContext context, String templateName,
      * @param jBallerinaBackend Backed to use.
      * @throws InvokerException If execution failed.
      */
-    protected void executeProject(JBallerinaBackend jBallerinaBackend, Project project,
-                                  Map<Identifier, Set<Identifier>> newImports)
-            throws InvokerException {
+    protected void executeProject(JBallerinaBackend jBallerinaBackend) throws InvokerException {
         if (bufferFile == null) {
             throw new UnsupportedOperationException("Buffer file must be set before execution");
         }
@@ -385,35 +370,9 @@ protected void executeProject(JBallerinaBackend jBallerinaBackend, Project proje
             // Main method class name is file name without extension
             String fileName = bufferFile.getName();
             String mainMethodClassName = fileName.substring(0, fileName.length() - TEMP_FILE_SUFFIX.length());
-            JarResolver jarResolver = jBallerinaBackend.jarResolver();
-            if (importClassLoader == null) {
-                importClassLoader = getClassLoaderWithRequiredJarFilesForExecutionWithoutMainJar(jBallerinaBackend,
-                                                                                                jarResolver);
-            }
-
-            ArrayList<URL> urlList = new ArrayList<>();
-            if (newImports.size() > 0) {
-                jarResolver.getJarFilePathsRequiredForExecution().stream()
-                        .filter(jarLibrary -> !jarLibrary.path().toString().contains("main"))
-                        .forEach(jarLibrary -> {
-                            try {
-                                urlList.add(new URL(jarLibrary.path().toUri().toURL().toString()));
-                            } catch (MalformedURLException e) {
-                                throw new RuntimeException(e);
-                            }
-                        });
-
-                URL[] urlArr = new URL[urlList.size()];
-                urlArr = urlList.toArray(urlArr);
-                importClassLoader = URLClassLoader.newInstance(urlArr, importClassLoader);
-            }
-
-            ModuleName moduleName = project.currentPackage().getDefaultModule().moduleName();
-            PlatformLibrary generatedJarLibrary = jBallerinaBackend.codeGeneratedLibrary(
-                    project.currentPackage().packageId(), moduleName);
-            URL u = new URL(generatedJarLibrary.path().toUri().toURL().toString());
-            classLoader = URLClassLoader.newInstance(new URL[]{u}, importClassLoader);
 
+            JarResolver jarResolver = jBallerinaBackend.jarResolver();
+            ClassLoader classLoader = jarResolver.getClassLoaderWithRequiredJarFilesForExecution();
             // First run configure initialization
             // TODO: (#28662) After configurables can be supported, change this to that file location
             invokeMethodDirectly(classLoader, CONFIGURE_INIT_CLASS_NAME, CONFIGURE_INIT_METHOD_NAME,
@@ -429,11 +388,15 @@ protected void executeProject(JBallerinaBackend jBallerinaBackend, Project proje
                 errorStream.println("fail: " + failErrorMessage);
             }
         } catch (InvokerPanicException panicError) {
+            List<String> stacktrace = Arrays.stream(panicError.getCause().getStackTrace())
+                    .filter(element -> !(element.toString().contains(MODULE_STATEMENT_METHOD_NAME) ||
+                                        element.toString().contains(MODULE_RUN_METHOD_NAME)))
+                    .collect(Collectors.toList())
+                    .stream().map(element -> "at " + element.getMethodName() + "()").collect(Collectors.toList());
             errorStream.println("panic: " + StringUtils.getErrorStringValue(panicError.getCause()));
+            stacktrace.forEach(errorStream::println);
             addErrorDiagnostic("Execution aborted due to unhandled runtime error.");
             throw panicError;
-        } catch (MalformedURLException e) {
-            throw new RuntimeException(e);
         }
     }
 
@@ -590,49 +553,4 @@ public File getBufferFile() throws IOException {
     protected PrintStream getErrorStream() {
         return System.err;
     }
-
-    /**
-     * Get classLoader without main jar.
-     *
-     * @param jBallerinaBackend jballerina backend.
-     * @param jarResolver jarResolver.
-     * @return class Loader without main jar.
-     */
-    public ClassLoader getClassLoaderWithRequiredJarFilesForExecutionWithoutMainJar(JBallerinaBackend jBallerinaBackend,
-                                                                                    JarResolver jarResolver) {
-
-        Collection<JarLibrary> jarLibraries = new ArrayList<>();
-        jarResolver.getJarFilePathsRequiredForExecution().stream()
-                .filter(jarLibrary -> !jarLibrary.path().toString().contains("main")).forEach(jarLibraries::add);
-
-        return createClassLoader(jBallerinaBackend, jarLibraries);
-    }
-
-    /**
-     * Create new classLoader.
-     *
-     * @param jBallerinaBackend jballerina backend.
-     * @param jarFiles required jar files to create classLoader.
-     * @return created classLoader.
-     */
-    private URLClassLoader createClassLoader(JBallerinaBackend jBallerinaBackend, Collection<JarLibrary> jarFiles) {
-        if (jBallerinaBackend.diagnosticResult().hasErrors()) {
-            throw new IllegalStateException("Cannot create a ClassLoader: this compilation has errors.");
-        }
-
-        List<URL> urlList = new ArrayList<>(jarFiles.size());
-        for (JarLibrary jarFile : jarFiles) {
-            try {
-                urlList.add(jarFile.path().toUri().toURL());
-            } catch (MalformedURLException e) {
-                // This path cannot get executed
-                throw new RuntimeException("Failed to create classloader with all jar files", e);
-            }
-        }
-
-        return AccessController.doPrivileged(
-                (PrivilegedAction<URLClassLoader>) () -> new URLClassLoader(urlList.toArray(new URL[0]),
-                        ClassLoader.getSystemClassLoader())
-        );
-    }
 }

ballerina-shell/modules/shell-core/src/main/java/io/ballerina/shell/invoker/classload/ClassLoadInvoker.java

@@ -288,7 +288,7 @@ public Optional<Object> execute(Optional<PackageCompilation> compilation) throws
         // If there are no declarations/variables, we can simply execute.
         if (noModuleDeclarations && noVariableDeclarations) {
             ClassLoadContext execContext = createVariablesExecutionContext(List.of(), executableSnippets, Map.of());
-            executeProject(execContext, EXECUTION_TEMPLATE_FILE, newImports);
+            executeProject(execContext, EXECUTION_TEMPLATE_FILE);
             return Optional.ofNullable(InvokerMemory.recall(contextId, CONTEXT_EXPR_VAR_NAME));
         }
 
@@ -323,7 +323,7 @@ public Optional<Object> execute(Optional<PackageCompilation> compilation) throws
         try {
             ClassLoadContext execContext = createVariablesExecutionContext(
                     variableDeclarations.keySet(), executableSnippets, allNewVariables);
-            executeProject(execContext, EXECUTION_TEMPLATE_FILE, newImports);
+            executeProject(execContext, EXECUTION_TEMPLATE_FILE);
             return Optional.ofNullable(InvokerMemory.recall(contextId, CONTEXT_EXPR_VAR_NAME));
         } catch (InvokerException e) {
             // Execution failed... Reverse all by deleting declarations.