Improve compile-time memory consumption for large projects

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/BuildCommand.java

@@ -71,6 +71,16 @@ public BuildCommand() {
         this.offline = true;
     }
 
+    BuildCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
+                 Boolean optimizeDependencyCompilation) {
+        this.projectPath = projectPath;
+        this.outStream = outStream;
+        this.errStream = errStream;
+        this.exitWhenFinish = exitWhenFinish;
+        this.optimizeDependencyCompilation = optimizeDependencyCompilation;
+        this.offline = true;
+    }
+
     BuildCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
                         boolean dumpBuildTime) {
         this.projectPath = projectPath;
@@ -198,6 +208,10 @@ public BuildCommand() {
             "generation")
     private String graalVMBuildOptions;
 
+    @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
+            description = "experimental memory optimization for large projects")
+    private Boolean optimizeDependencyCompilation;
+
     @Override
     public void execute() {
         long start = 0;
@@ -317,7 +331,8 @@ private BuildOptions constructBuildOptions() {
                 .setNativeImage(nativeImage)
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
                 .setGraalVMBuildOptions(graalVMBuildOptions)
-                .setShowDependencyDiagnostics(showDependencyDiagnostics);
+                .setShowDependencyDiagnostics(showDependencyDiagnostics)
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/CommandUtil.java

@@ -21,6 +21,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
+import io.ballerina.projects.BuildOptions;
 import io.ballerina.projects.JBallerinaBackend;
 import io.ballerina.projects.JvmTarget;
 import io.ballerina.projects.Package;
@@ -1151,12 +1152,14 @@
      * @param orgName org name of the dependent package
      * @param packageName name of the dependent package
      * @param version version of the dependent package
+     * @param buildOptions build options {sticky, offline}
      * @return true if the dependent package compilation has errors
      */
-    static boolean pullDependencyPackages(String orgName, String packageName, String version) {
+    static boolean pullDependencyPackages(String orgName, String packageName, String version,
+                                          BuildOptions buildOptions, String repository) {
         Path ballerinaUserHomeDirPath = ProjectUtils.createAndGetHomeReposPath();
         Path centralRepositoryDirPath = ballerinaUserHomeDirPath.resolve(ProjectConstants.REPOSITORIES_DIR)
-                .resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME);
+                .resolve(repository);
         Path balaDirPath = centralRepositoryDirPath.resolve(ProjectConstants.BALA_DIR_NAME);
         Path balaPath = ProjectUtils.getPackagePath(balaDirPath, orgName, packageName, version);
         String ballerinaShortVersion = RepoUtils.getBallerinaShortVersion();
@@ -1165,7 +1168,7 @@
 
         ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
         defaultBuilder.addCompilationCacheFactory(new FileSystemCache.FileSystemCacheFactory(cacheDir));
-        BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);
+        BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath, buildOptions);
 
         // Delete package cache if available
         Path packageCacheDir = cacheDir.resolve(orgName).resolve(packageName).resolve(version);

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/DocCommand.java

@@ -114,6 +114,10 @@ public DocCommand() {
             "generated by the dependencies")
     private Boolean showDependencyDiagnostics;
 
+    @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
+            description = "experimental memory optimization for large projects")
+    private Boolean optimizeDependencyCompilation;
+
     @Override
     public void execute() {
         if (this.helpFlag) {
@@ -227,8 +231,8 @@ private BuildOptions constructBuildOptions() {
                 .setTestReport(false)
                 .setObservabilityIncluded(false)
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
-                .setShowDependencyDiagnostics(showDependencyDiagnostics);
-
+                .setShowDependencyDiagnostics(showDependencyDiagnostics)
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PackCommand.java

@@ -93,6 +93,10 @@ public class PackCommand implements BLauncherCmd {
             "generated by the dependencies")
     private Boolean showDependencyDiagnostics;
 
+    @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
+            description = "experimental memory optimization for large projects")
+    private Boolean optimizeDependencyCompilation;
+
     public PackCommand() {
         this.projectPath = Paths.get(System.getProperty(ProjectConstants.USER_DIR));
         this.outStream = System.out;
@@ -111,6 +115,17 @@ public PackCommand() {
         this.offline = true;
     }
 
+    PackCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
+                boolean skipCopyLibsFromDist, Boolean optimizeDependencyCompilation) {
+        this.projectPath = projectPath;
+        this.outStream = outStream;
+        this.errStream = errStream;
+        this.exitWhenFinish = exitWhenFinish;
+        this.skipCopyLibsFromDist = skipCopyLibsFromDist;
+        this.optimizeDependencyCompilation = optimizeDependencyCompilation;
+        this.offline = true;
+    }
+
     PackCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
                        boolean skipCopyLibsFromDist, Path targetDir) {
         this.projectPath = projectPath;
@@ -269,7 +284,8 @@ private BuildOptions constructBuildOptions() {
                 .setConfigSchemaGen(configSchemaGen)
                 .setEnableCache(enableCache)
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
-                .setShowDependencyDiagnostics(showDependencyDiagnostics);
+                .setShowDependencyDiagnostics(showDependencyDiagnostics)
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/ProfileCommand.java

@@ -91,6 +91,10 @@ public class ProfileCommand implements BLauncherCmd {
             "generated by the dependencies")
     private Boolean showDependencyDiagnostics;
 
+    @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
+            description = "experimental memory optimization for large projects")
+    private Boolean optimizeDependencyCompilation;
+
     private static final String PROFILE_CMD = "bal profile [--debug <port>] [<ballerina-file | package-path>]\n ";
 
     public ProfileCommand() {
@@ -231,7 +235,8 @@ private BuildOptions constructBuildOptions() {
                 .setTestReport(false)
                 .setConfigSchemaGen(configSchemaGen)
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
-                .setShowDependencyDiagnostics(showDependencyDiagnostics);
+                .setShowDependencyDiagnostics(showDependencyDiagnostics)
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

@@ -21,6 +21,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import io.ballerina.cli.BLauncherCmd;
+import io.ballerina.projects.BuildOptions;
 import io.ballerina.projects.JvmTarget;
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.SemanticVersion;
@@ -53,6 +54,7 @@
 import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
 import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
+import static io.ballerina.projects.util.ProjectConstants.LOCAL_REPOSITORY_NAME;
 import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
 import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
 import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
@@ -72,6 +74,7 @@
     private static final String USAGE_TEXT =
             "bal pull {<org-name>/<package-name> | <org-name>/<package-name>:<version>}";
 
+    private final PrintStream outStream;
     private final PrintStream errStream;
     private final boolean exitWhenFinish;
 
@@ -87,14 +90,23 @@
     @CommandLine.Option(names = "--repository")
     private String repositoryName;
 
+    @CommandLine.Option(names = "--sticky", hidden = true, defaultValue = "true")
+    private boolean sticky;
+
+    @CommandLine.Option(names = "--offline", hidden = true)
+    private boolean offline;
+
     public PullCommand() {
+        this.outStream = System.out;
         this.errStream = System.err;
         this.exitWhenFinish = true;
     }
 
     public PullCommand(PrintStream errStream, boolean exitWhenFinish) {
+        this.outStream = errStream;
         this.errStream = errStream;
         this.exitWhenFinish = exitWhenFinish;
+        this.repositoryName = null;
     }
 
     @Override
@@ -132,7 +144,7 @@
         // Get org name
         String[] moduleInfo = resourceName.split("/");
         if (moduleInfo.length != 2) {
-            CommandUtil.printError(errStream, "invalid package name. Provide the package name with the organization.",
+            CommandUtil.printError(errStream, "invalid package. Provide the package name with the organization.",
                                    USAGE_TEXT, false);
             CommandUtil.exitError(this.exitWhenFinish);
             return;
@@ -149,7 +161,7 @@
             packageName = moduleNameAndVersion;
             version = Names.EMPTY.getValue();
         } else {
-            CommandUtil.printError(errStream, "invalid package name. Provide the package name with the organization.",
+            CommandUtil.printError(errStream, "invalid package. Provide the package name with the organization.",
                                    USAGE_TEXT, false);
             CommandUtil.exitError(this.exitWhenFinish);
             return;
@@ -186,13 +198,77 @@
             settings = Settings.from();
         }
 
+        if (repositoryName == null) {
+            repositoryName = ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME;
+            version = pullFromCentral(settings, orgName, packageName, version);
+        } else if (!LOCAL_REPOSITORY_NAME.equals(repositoryName)) {
+            pullFromMavenRepo(settings, orgName, packageName, version);
+        }
+
+        if (!resolveDependencies(orgName, packageName, version)) {
+            CommandUtil.exitError(this.exitWhenFinish);
+            return;
+        }
+
+        if (this.exitWhenFinish) {
+            Runtime.getRuntime().exit(0);
+        }
+
+    }
+
+    private String pullFromCentral(Settings settings, String orgName, String packageName, String version) {
+        Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
+                .resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
+                .resolve(ProjectConstants.BALA_DIR_NAME)
+                .resolve(orgName).resolve(packageName);
+
+        if (!version.equals(Names.EMPTY.getValue()) && Files.exists(packagePathInBalaCache.resolve(version))) {
+            outStream.println("Package already exists.\n");
+        }
+        // create directory path in bala cache
+        try {
+            createDirectories(packagePathInBalaCache);
+        } catch (IOException e) {
+            CommandUtil.exitError(this.exitWhenFinish);
+            throw createLauncherException(
+                    "unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
+        }
+
+        CommandUtil.setPrintStream(errStream);
+        String supportedPlatform = Arrays.stream(JvmTarget.values())
+                .map(JvmTarget::code)
+                .collect(Collectors.joining(","));
+        CentralAPIClient client;
+        try {
+            client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
+                    initializeProxy(settings.getProxy()), settings.getProxy().username(),
+                    settings.getProxy().password(), getAccessTokenOfCLI(settings),
+                    settings.getCentral().getConnectTimeout(),
+                    settings.getCentral().getReadTimeout(), settings.getCentral().getWriteTimeout(),
+                    settings.getCentral().getCallTimeout(), settings.getCentral().getMaxRetries());
+            client.pullPackage(orgName, packageName, version, packagePathInBalaCache, supportedPlatform,
+                    RepoUtils.getBallerinaVersion(), false);
+            if (version.equals(Names.EMPTY.getValue())) {
+                List<String> versions = client.getPackageVersions(orgName, packageName, supportedPlatform,
+                        RepoUtils.getBallerinaVersion());
+                version = CommandUtil.getLatestVersion(versions);
+            }
+        } catch (PackageAlreadyExistsException e) {
+            outStream.println("Package already exists.\n");
+            version = e.version();
+        } catch (CentralClientException e) {
+            errStream.println("package not found: " + orgName + "/" + packageName);
+            CommandUtil.exitError(this.exitWhenFinish);
+        }
+        return version;
+    }
+
+    private void pullFromMavenRepo(Settings settings, String orgName, String packageName, String version) {
         Repository targetRepository = null;
-        if (repositoryName != null) {
-            for (Repository repository : settings.getRepositories()) {
-                if (repositoryName.equals(repository.id())) {
-                    targetRepository = repository;
-                    break;
-                }
+        for (Repository repository : settings.getRepositories()) {
+            if (repositoryName.equals(repository.id())) {
+                targetRepository = repository;
+                break;
             }
         }
 
@@ -218,7 +294,8 @@
             Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
                     .resolve(ProjectConstants.REPOSITORIES_DIR)
                     .resolve(targetRepository.id())
-                    .resolve(ProjectConstants.BALA_DIR_NAME);
+                    .resolve(ProjectConstants.BALA_DIR_NAME)
+                    .resolve(orgName).resolve(packageName).resolve(version);
 
             try {
                 Path tmpDownloadDirectory = Files.createTempDirectory("ballerina-" + System.nanoTime());
@@ -233,8 +310,7 @@
                 try (BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8)) {
                     JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
                     String platform = resultObj.get(PLATFORM).getAsString();
-                    Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
-                            .resolve(version).resolve(platform);
+                    Path actualBalaPath = mavenBalaCachePath.resolve(platform);
                     org.apache.commons.io.FileUtils.copyDirectory(temporaryExtractionPath.toFile(),
                             actualBalaPath.toFile());
                 }
@@ -247,56 +323,24 @@
             }
             PrintStream out = System.out;
             out.println("Successfully pulled the package from the custom repository.");
-            return;
-        }
-
-        Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
-                .resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
-                .resolve(ProjectConstants.BALA_DIR_NAME)
-                .resolve(orgName).resolve(packageName);
-        // create directory path in bala cache
-        try {
-            createDirectories(packagePathInBalaCache);
-        } catch (IOException e) {
-            CommandUtil.exitError(this.exitWhenFinish);
-            throw createLauncherException(
-                    "unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
         }
+    }
 
+    private boolean resolveDependencies(String orgName, String packageName, String version) {
         CommandUtil.setPrintStream(errStream);
-        String supportedPlatform = Arrays.stream(JvmTarget.values())
-                .map(JvmTarget::code)
-                .collect(Collectors.joining(","));
         try {
-            CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
-                    initializeProxy(settings.getProxy()), settings.getProxy().username(),
-                    settings.getProxy().password(), getAccessTokenOfCLI(settings),
-                    settings.getCentral().getConnectTimeout(),
-                    settings.getCentral().getReadTimeout(), settings.getCentral().getWriteTimeout(),
-                    settings.getCentral().getCallTimeout(), settings.getCentral().getMaxRetries());
-            client.pullPackage(orgName, packageName, version, packagePathInBalaCache, supportedPlatform,
-                    RepoUtils.getBallerinaVersion(), false);
-            if (version.equals(Names.EMPTY.getValue())) {
-                List<String> versions = client.getPackageVersions(orgName, packageName, supportedPlatform,
-                        RepoUtils.getBallerinaVersion());
-                version = CommandUtil.getLatestVersion(versions);
-            }
-            boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(orgName, packageName, version);
+            BuildOptions buildOptions = BuildOptions.builder().setSticky(sticky).setOffline(offline).build();
+            boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(
+                    orgName, packageName, version, buildOptions, repositoryName);
             if (hasCompilationErrors) {
                 CommandUtil.printError(this.errStream, "compilation contains errors", null, false);
-                CommandUtil.exitError(this.exitWhenFinish);
-                return;
+                return false;
             }
-        } catch (PackageAlreadyExistsException e) {
-            errStream.println(e.getMessage());
-            CommandUtil.exitError(this.exitWhenFinish);
-        } catch (CentralClientException e) {
-            errStream.println("package not found: " + orgName + "/" + packageName);
-            CommandUtil.exitError(this.exitWhenFinish);
-        }
-        if (this.exitWhenFinish) {
-            Runtime.getRuntime().exit(0);
+        } catch (ProjectException e) {
+            CommandUtil.printError(this.errStream,
+                    "error occurred while resolving dependencies, reason: " + e.getMessage(), null, false);
         }
+        return true;
     }
 
     @Override