Merge pull request #41050 from AzeemMuzammil/fb-format-import-org

Add Import organization support

misc/formatter/modules/formatter-core/build.gradle

@@ -27,6 +27,8 @@ dependencies {
     implementation project(':ballerina-tools-api')
     implementation project(':ballerina-parser')
     implementation project(':ballerina-lang')
+    implementation "org.apache.commons:commons-lang3:${project.apacheCommonsLang3Version}"
+
     testImplementation 'com.google.code.gson:gson'
     testImplementation 'org.testng:testng'
 }

misc/formatter/modules/formatter-core/src/main/java/module-info.java

@@ -2,6 +2,8 @@
     requires io.ballerina.lang;
     requires io.ballerina.parser;
     requires io.ballerina.tools.api;
+    requires org.apache.commons.lang3;
     requires org.slf4j;
+
     exports org.ballerinalang.formatter.core;
 }

misc/formatter/modules/formatter-core/src/main/java/org/ballerinalang/formatter/core/FormatterUtils.java

@@ -15,8 +15,13 @@
  */
 package org.ballerinalang.formatter.core;
 
+import io.ballerina.compiler.syntax.tree.ImportDeclarationNode;
 import io.ballerina.compiler.syntax.tree.Node;
 import io.ballerina.tools.text.LineRange;
+import org.apache.commons.lang3.builder.CompareToBuilder;
+
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Class that contains the util functions used by the formatting tree modifier.
@@ -60,4 +65,18 @@ static boolean isInlineRange(Node node, LineRange lineRange) {
 
         return true;
     }
+
+    /**
+     * Sort ImportDeclaration nodes based on orgName and the moduleName in-place.
+     *
+     * @param importDeclarationNodes ImportDeclarations nodes
+     */
+    static void sortImportDeclarations(List<ImportDeclarationNode> importDeclarationNodes) {
+        importDeclarationNodes.sort((node1, node2) -> new CompareToBuilder()
+                .append(node1.orgName().isPresent() ? node1.orgName().get().orgName().text() : "",
+                        node2.orgName().isPresent() ? node2.orgName().get().orgName().text() : "")
+                .append(node1.moduleName().stream().map(node -> node.toString().trim()).collect(Collectors.joining()),
+                        node2.moduleName().stream().map(node -> node.toString().trim()).collect(Collectors.joining()))
+                .toComparison());
+    }
 }

misc/formatter/modules/formatter-core/src/main/java/org/ballerinalang/formatter/core/FormattingOptions.java

@@ -32,13 +32,17 @@ public class FormattingOptions {
 
     private ForceFormattingOptions forceFormattingOptions;
 
-    FormattingOptions(int tabSize, String wsCharacter, int columnLimit, boolean lineWrapping,
-                      ForceFormattingOptions forceFormattingOptions) {
+    private ImportFormattingOptions importFormattingOptions;
+
+    private FormattingOptions(int tabSize, String wsCharacter, int columnLimit, boolean lineWrapping,
+                              ForceFormattingOptions forceFormattingOptions,
+                              ImportFormattingOptions importFormattingOptions) {
         this.tabSize = tabSize;
         this.wsCharacter = wsCharacter;
         this.columnLimit = columnLimit;
         this.lineWrapping = lineWrapping;
         this.forceFormattingOptions = forceFormattingOptions;
+        this.importFormattingOptions = importFormattingOptions;
     }
 
     /**
@@ -140,6 +144,10 @@ public ForceFormattingOptions getForceFormattingOptions() {
         return forceFormattingOptions;
     }
 
+    public ImportFormattingOptions getImportFormattingOptions() {
+        return importFormattingOptions;
+    }
+
     public static FormattingOptionsBuilder builder() {
         return new FormattingOptionsBuilder();
     }
@@ -155,6 +163,7 @@ public static class FormattingOptionsBuilder {
         private int columnLimit = 120;
         private boolean lineWrapping = false;
         private ForceFormattingOptions forceFormattingOptions = ForceFormattingOptions.builder().build();
+        private ImportFormattingOptions importFormattingOptions = ImportFormattingOptions.builder().build();
 
         public FormattingOptions.FormattingOptionsBuilder setTabSize(int tabSize) {
             this.tabSize = tabSize;
@@ -182,8 +191,15 @@ public FormattingOptions.FormattingOptionsBuilder setForceFormattingOptions(
             return this;
         }
 
+        public FormattingOptions.FormattingOptionsBuilder setImportFormattingOptions(
+                ImportFormattingOptions importFormattingOptions) {
+            this.importFormattingOptions = importFormattingOptions;
+            return this;
+        }
+
         public FormattingOptions build() {
-            return new FormattingOptions(tabSize, wsCharacter, columnLimit, lineWrapping, forceFormattingOptions);
+            return new FormattingOptions(tabSize, wsCharacter, columnLimit, lineWrapping, forceFormattingOptions,
+                    importFormattingOptions);
         }
     }
 }

misc/formatter/modules/formatter-core/src/main/java/org/ballerinalang/formatter/core/FormattingTreeModifier.java

@@ -252,8 +252,10 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import static org.ballerinalang.formatter.core.FormatterUtils.isInlineRange;
+import static org.ballerinalang.formatter.core.FormatterUtils.sortImportDeclarations;
 
 /**
  * A formatter implementation that updates the minutiae of a given tree according to the ballerina formatting
@@ -284,7 +286,7 @@ public FormattingTreeModifier(FormattingOptions options, LineRange lineRange) {
 
     @Override
     public ModulePartNode transform(ModulePartNode modulePartNode) {
-        NodeList<ImportDeclarationNode> imports = formatNodeList(modulePartNode.imports(), 0, 1, 0, 2);
+        NodeList<ImportDeclarationNode> imports = sortAndGroupImportDeclarationNodes(modulePartNode.imports());
         NodeList<ModuleMemberDeclarationNode> members = formatModuleMembers(modulePartNode.members());
         Token eofToken = formatToken(modulePartNode.eofToken(), 0, 0);
         return modulePartNode.modify(imports, members, eofToken);
@@ -643,7 +645,10 @@ public RecordFieldWithDefaultValueNode transform(RecordFieldWithDefaultValueNode
 
     @Override
     public ImportDeclarationNode transform(ImportDeclarationNode importDeclarationNode) {
+        boolean prevPreservedNewLine = env.hasPreservedNewline;
+        setPreserveNewline(false);
         Token importKeyword = formatToken(importDeclarationNode.importKeyword(), 1, 0);
+        setPreserveNewline(prevPreservedNewLine);
         boolean hasPrefix = importDeclarationNode.prefix().isPresent();
         ImportOrgNameNode orgName = formatNode(importDeclarationNode.orgName().orElse(null), 0, 0);
         SeparatedNodeList<IdentifierToken> moduleNames = formatSeparatedNodeList(importDeclarationNode.moduleName(),
@@ -4283,6 +4288,11 @@ private MinutiaeList getLeadingMinutiae(Token token) {
             addWhitespace(env.currentIndentation, leadingMinutiae);
         }
 
+        if (leadingMinutiae.size() > 0 &&
+                leadingMinutiae.get(leadingMinutiae.size() - 1).kind().equals(SyntaxKind.COMMENT_MINUTIAE)) {
+            leadingMinutiae.add(getNewline());
+        }
+
         MinutiaeList newLeadingMinutiaeList = NodeFactory.createMinutiaeList(leadingMinutiae);
         preserveIndentation(false);
         return newLeadingMinutiaeList;
@@ -4663,4 +4673,44 @@ private boolean hasTrailingNL(Token token) {
         }
         return false;
     }
+
+    private NodeList<ImportDeclarationNode> sortAndGroupImportDeclarationNodes(
+            NodeList<ImportDeclarationNode> importDeclarationNodes) {
+        // moduleImports would collect only module level imports if grouping is enabled,
+        // and would collect all imports otherwise
+        List<ImportDeclarationNode> moduleImports = new ArrayList<>();
+        List<ImportDeclarationNode> stdLibImports = new ArrayList<>();
+        List<ImportDeclarationNode> thirdPartyImports = new ArrayList<>();
+
+        for (ImportDeclarationNode importDeclarationNode : importDeclarationNodes) {
+            if (importDeclarationNode.orgName().isEmpty() || !options.getImportFormattingOptions().getGroupImports()) {
+                moduleImports.add(importDeclarationNode);
+            } else {
+                if (List.of("ballerina", "ballerinax")
+                        .contains(importDeclarationNode.orgName().get().orgName().text())) {
+                    stdLibImports.add(importDeclarationNode);
+                } else {
+                    thirdPartyImports.add(importDeclarationNode);
+                }
+            }
+        }
+        if (options.getImportFormattingOptions().getSortImports()) {
+            sortImportDeclarations(moduleImports);
+            sortImportDeclarations(stdLibImports);
+            sortImportDeclarations(thirdPartyImports);
+        }
+
+        NodeList<ImportDeclarationNode> moduleImportNodes =
+                formatNodeList(NodeFactory.createNodeList(moduleImports), 0, 1, 0, 2);
+        NodeList<ImportDeclarationNode> stdLibImportNodes =
+                formatNodeList(NodeFactory.createNodeList(stdLibImports), 0, 1, 0, 2);
+        NodeList<ImportDeclarationNode> thirdPartyImportNodes =
+                formatNodeList(NodeFactory.createNodeList(thirdPartyImports), 0, 1, 0, 2);
+
+        List<ImportDeclarationNode> imports = new ArrayList<>();
+        imports.addAll(moduleImportNodes.stream().collect(Collectors.toList()));
+        imports.addAll(stdLibImportNodes.stream().collect(Collectors.toList()));
+        imports.addAll(thirdPartyImportNodes.stream().collect(Collectors.toList()));
+        return NodeFactory.createNodeList(imports);
+    }
 }

misc/formatter/modules/formatter-core/src/main/java/org/ballerinalang/formatter/core/ImportFormattingOptions.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2023, WSO2 LLC. (http://wso2.com) All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.ballerinalang.formatter.core;
+
+/**
+ * A model for formatting and optimizing imports by the API user, that could be passed onto the formatter.
+ *
+ * @since 2201.8.0
+ */
+public class ImportFormattingOptions {
+    private boolean groupImports;
+    private boolean sortImports;
+    private boolean removeUnusedImports;
+
+    private ImportFormattingOptions(boolean groupImports, boolean sortImports, boolean removeUnusedImports) {
+        this.groupImports = groupImports;
+        this.sortImports = sortImports;
+        this.removeUnusedImports = removeUnusedImports;
+    }
+
+    public boolean getGroupImports() {
+        return groupImports;
+    }
+
+    public boolean getSortImports() {
+        return sortImports;
+    }
+
+    public boolean getRemoveUnusedImports() {
+        return removeUnusedImports;
+    }
+
+    public static ImportFormattingOptions.ImportFormattingOptionsBuilder builder() {
+        return new ImportFormattingOptions.ImportFormattingOptionsBuilder();
+    }
+
+    /**
+     * A builder for the {@code ImportFormattingOptions}.
+     */
+    public static class ImportFormattingOptionsBuilder {
+        private boolean groupImports = true;
+        private boolean sortImports = true;
+        private boolean removeUnusedImports = false;
+
+        public ImportFormattingOptions.ImportFormattingOptionsBuilder setGroupImports(boolean groupImports) {
+            this.groupImports = groupImports;
+            return this;
+        }
+
+        public ImportFormattingOptions.ImportFormattingOptionsBuilder setSortImports(boolean sortImports) {
+            this.sortImports = sortImports;
+            return  this;
+        }
+
+        public ImportFormattingOptions.ImportFormattingOptionsBuilder setRemoveUnusedImports(
+                boolean removeUnusedImports) {
+            this.removeUnusedImports = removeUnusedImports;
+            return this;
+        }
+
+        public ImportFormattingOptions build() {
+            return new ImportFormattingOptions(groupImports, sortImports, removeUnusedImports);
+        }
+    }
+}

misc/formatter/modules/formatter-core/src/test/java/org/ballerinalang/formatter/core/FormatterTest.java

@@ -26,6 +26,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -83,6 +84,21 @@ public void testWithOptions(String source, String sourcePath) throws IOException
         }
     }
 
+    public void testWithCustomOptions(String source, String sourcePath, FormattingOptions formattingOptions)
+            throws IOException {
+        Path assertFilePath = Paths.get(resourceDirectory.toString(), sourcePath, ASSERT_DIR, source);
+        Path sourceFilePath = Paths.get(resourceDirectory.toString(), sourcePath, SOURCE_DIR, source);
+        String content = getSourceText(sourceFilePath);
+        TextDocument textDocument = TextDocuments.from(content);
+        SyntaxTree syntaxTree = SyntaxTree.from(textDocument);
+        try {
+            SyntaxTree newSyntaxTree = Formatter.format(syntaxTree, formattingOptions);
+            Assert.assertEquals(newSyntaxTree.toSourceCode(), getSourceText(assertFilePath));
+        } catch (FormatterException e) {
+            Assert.fail(e.getMessage(), e);
+        }
+    }
+
     /**
      * Test the formatting functionality for parser test cases.
      *
@@ -127,6 +143,16 @@ public Object[][] testSubset() {
         return new Object[0][];
     }
 
+    /**
+     * Specify the file names to be tested with specific tests during the test execution.
+     *
+     * @return Test scenarios for execution
+     */
+    @DataProvider(name = "test-file-provider-custom")
+    public Object[][] dataProviderWithCustomTests(Method testName) {
+        return new Object[0][];
+    }
+
     /**
      * Returns the directory path inside resources which holds the test files.
      *

misc/formatter/modules/formatter-core/src/test/java/org/ballerinalang/formatter/core/ParserTestFormatter.java

@@ -58,6 +58,7 @@ public List<String> skipList() {
                 "minutiae_test_03.bal",
                 "minutiae_test_04.bal",
                 "minutiae_test_05.bal",
+                "minutiae_test_04_with_no_newlines.bal",
                 "minutiae_test_05_with_no_newlines.bal",
                 "invalid_token_minutiae_test_01.bal",
                 "invalid_token_minutiae_test_02.bal",
@@ -93,6 +94,8 @@ public List<String> skipList() {
                 "query_expr_source_126.bal", "match_stmt_source_21.bal",
                 "func_params_source_27.bal",
 
+                "separated_node_list_import_decl.bal", "node_location_test_03.bal",
+
                 // parser tests with syntax errors that cannot be handled by the formatter
                 "worker_decl_source_03.bal", "worker_decl_source_05.bal", "invalid_identifier_source_01.bal",
                 "ambiguity_source_23.bal", "ambiguity_source_09.bal", "ambiguity_source_18.bal",
@@ -111,7 +114,7 @@ public List<String> skipList() {
                 "tuple_type_source_04.bal", "tuple_type_source_06.bal", "trivia_source_02.bal",
                 "enum_decl_source_05.bal", "enum_decl_source_08.bal", "enum_decl_source_09.bal",
                 "service_decl_source_09.bal", "service_decl_source_15.bal", "service_decl_source_03.bal",
-                 "service_decl_source_12.bal", "service_decl_source_10.bal", "service_decl_source_04.bal",
+                "service_decl_source_12.bal", "service_decl_source_10.bal", "service_decl_source_04.bal",
                 "service_decl_source_11.bal", "import_decl_source_19.bal", "import_decl_source_20.bal",
                 "import_decl_source_21.bal", "import_decl_source_23.bal", "import_decl_source_22.bal",
                 "import_decl_source_06.bal", "import_decl_source_04.bal", "import_decl_source_10.bal",