Skip to content

Commit

Permalink
Merge pull request #43239 from azinneera/prj_api_oom10x
Browse files Browse the repository at this point in the history 
Improve compile-time memory consumption for large projects
  • Loading branch information
@azinneera
azinneera authored Aug 5, 2024
2 parents d7bdc12 + 9651394 commit aabac46
Show file tree
Hide file tree
Showing 31 changed files with 534 additions and 251 deletions.
17 changes: 16 additions & 1 deletion cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/BuildCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ public BuildCommand() {
this.offline = true;
}

BuildCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
Boolean optimizeDependencyCompilation) {
this.projectPath = projectPath;
this.outStream = outStream;
this.errStream = errStream;
this.exitWhenFinish = exitWhenFinish;
this.optimizeDependencyCompilation = optimizeDependencyCompilation;
this.offline = true;
}

BuildCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
boolean dumpBuildTime) {
this.projectPath = projectPath;
Expand Down Expand Up @@ -198,6 +208,10 @@ public BuildCommand() {
"generation")
private String graalVMBuildOptions;

@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
description = "experimental memory optimization for large projects")
private Boolean optimizeDependencyCompilation;

@Override
public void execute() {
long start = 0;
Expand Down Expand Up @@ -317,7 +331,8 @@ private BuildOptions constructBuildOptions() {
.setNativeImage(nativeImage)
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
.setGraalVMBuildOptions(graalVMBuildOptions)
.setShowDependencyDiagnostics(showDependencyDiagnostics);
.setShowDependencyDiagnostics(showDependencyDiagnostics)
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);

if (targetDir != null) {
buildOptionsBuilder.targetDir(targetDir.toString());
Expand Down
9 changes: 6 additions & 3 deletions cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/CommandUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.ballerina.projects.BuildOptions;
import io.ballerina.projects.JBallerinaBackend;
import io.ballerina.projects.JvmTarget;
import io.ballerina.projects.Package;
Expand Down Expand Up @@ -1151,12 +1152,14 @@ static String getLatestVersion(List<String> versions) {
* @param orgName org name of the dependent package
* @param packageName name of the dependent package
* @param version version of the dependent package
* @param buildOptions build options {sticky, offline}
* @return true if the dependent package compilation has errors
*/
static boolean pullDependencyPackages(String orgName, String packageName, String version) {
static boolean pullDependencyPackages(String orgName, String packageName, String version,
BuildOptions buildOptions, String repository) {
Path ballerinaUserHomeDirPath = ProjectUtils.createAndGetHomeReposPath();
Path centralRepositoryDirPath = ballerinaUserHomeDirPath.resolve(ProjectConstants.REPOSITORIES_DIR)
.resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME);
.resolve(repository);
Path balaDirPath = centralRepositoryDirPath.resolve(ProjectConstants.BALA_DIR_NAME);
Path balaPath = ProjectUtils.getPackagePath(balaDirPath, orgName, packageName, version);
String ballerinaShortVersion = RepoUtils.getBallerinaShortVersion();
Expand All @@ -1165,7 +1168,7 @@ static boolean pullDependencyPackages(String orgName, String packageName, String

ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
defaultBuilder.addCompilationCacheFactory(new FileSystemCache.FileSystemCacheFactory(cacheDir));
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath, buildOptions);

// Delete package cache if available
Path packageCacheDir = cacheDir.resolve(orgName).resolve(packageName).resolve(version);
Expand Down
8 changes: 6 additions & 2 deletions cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/DocCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ public DocCommand() {
"generated by the dependencies")
private Boolean showDependencyDiagnostics;

@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
description = "experimental memory optimization for large projects")
private Boolean optimizeDependencyCompilation;

@Override
public void execute() {
if (this.helpFlag) {
Expand Down Expand Up @@ -227,8 +231,8 @@ private BuildOptions constructBuildOptions() {
.setTestReport(false)
.setObservabilityIncluded(false)
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
.setShowDependencyDiagnostics(showDependencyDiagnostics);

.setShowDependencyDiagnostics(showDependencyDiagnostics)
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);

if (targetDir != null) {
buildOptionsBuilder.targetDir(targetDir.toString());
Expand Down
18 changes: 17 additions & 1 deletion cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PackCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ public class PackCommand implements BLauncherCmd {
"generated by the dependencies")
private Boolean showDependencyDiagnostics;

@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
description = "experimental memory optimization for large projects")
private Boolean optimizeDependencyCompilation;

public PackCommand() {
this.projectPath = Paths.get(System.getProperty(ProjectConstants.USER_DIR));
this.outStream = System.out;
Expand All @@ -111,6 +115,17 @@ public PackCommand() {
this.offline = true;
}

PackCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
boolean skipCopyLibsFromDist, Boolean optimizeDependencyCompilation) {
this.projectPath = projectPath;
this.outStream = outStream;
this.errStream = errStream;
this.exitWhenFinish = exitWhenFinish;
this.skipCopyLibsFromDist = skipCopyLibsFromDist;
this.optimizeDependencyCompilation = optimizeDependencyCompilation;
this.offline = true;
}

PackCommand(Path projectPath, PrintStream outStream, PrintStream errStream, boolean exitWhenFinish,
boolean skipCopyLibsFromDist, Path targetDir) {
this.projectPath = projectPath;
Expand Down Expand Up @@ -269,7 +284,8 @@ private BuildOptions constructBuildOptions() {
.setConfigSchemaGen(configSchemaGen)
.setEnableCache(enableCache)
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
.setShowDependencyDiagnostics(showDependencyDiagnostics);
.setShowDependencyDiagnostics(showDependencyDiagnostics)
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);

if (targetDir != null) {
buildOptionsBuilder.targetDir(targetDir.toString());
Expand Down
7 changes: 6 additions & 1 deletion cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/ProfileCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,10 @@ public class ProfileCommand implements BLauncherCmd {
"generated by the dependencies")
private Boolean showDependencyDiagnostics;

@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
description = "experimental memory optimization for large projects")
private Boolean optimizeDependencyCompilation;

private static final String PROFILE_CMD = "bal profile [--debug <port>] [<ballerina-file | package-path>]\n ";

public ProfileCommand() {
Expand Down Expand Up @@ -231,7 +235,8 @@ private BuildOptions constructBuildOptions() {
.setTestReport(false)
.setConfigSchemaGen(configSchemaGen)
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
.setShowDependencyDiagnostics(showDependencyDiagnostics);
.setShowDependencyDiagnostics(showDependencyDiagnostics)
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);

if (targetDir != null) {
buildOptionsBuilder.targetDir(targetDir.toString());
Expand Down
150 changes: 97 additions & 53 deletions cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.ballerina.cli.BLauncherCmd;
import io.ballerina.projects.BuildOptions;
import io.ballerina.projects.JvmTarget;
import io.ballerina.projects.ProjectException;
import io.ballerina.projects.SemanticVersion;
Expand Down Expand Up @@ -53,6 +54,7 @@
import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
import static io.ballerina.projects.util.ProjectConstants.LOCAL_REPOSITORY_NAME;
import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
Expand All @@ -72,6 +74,7 @@ public class PullCommand implements BLauncherCmd {
private static final String USAGE_TEXT =
"bal pull {<org-name>/<package-name> | <org-name>/<package-name>:<version>}";

private final PrintStream outStream;
private final PrintStream errStream;
private final boolean exitWhenFinish;

Expand All @@ -87,14 +90,23 @@ public class PullCommand implements BLauncherCmd {
@CommandLine.Option(names = "--repository")
private String repositoryName;

@CommandLine.Option(names = "--sticky", hidden = true, defaultValue = "true")
private boolean sticky;

@CommandLine.Option(names = "--offline", hidden = true)
private boolean offline;

public PullCommand() {
this.outStream = System.out;
this.errStream = System.err;
this.exitWhenFinish = true;
}

public PullCommand(PrintStream errStream, boolean exitWhenFinish) {
this.outStream = errStream;
this.errStream = errStream;
this.exitWhenFinish = exitWhenFinish;
this.repositoryName = null;
}

@Override
Expand Down Expand Up @@ -132,7 +144,7 @@ public void execute() {
// Get org name
String[] moduleInfo = resourceName.split("/");
if (moduleInfo.length != 2) {
CommandUtil.printError(errStream, "invalid package name. Provide the package name with the organization.",
CommandUtil.printError(errStream, "invalid package. Provide the package name with the organization.",
USAGE_TEXT, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
Expand All @@ -149,7 +161,7 @@ public void execute() {
packageName = moduleNameAndVersion;
version = Names.EMPTY.getValue();
} else {
CommandUtil.printError(errStream, "invalid package name. Provide the package name with the organization.",
CommandUtil.printError(errStream, "invalid package. Provide the package name with the organization.",
USAGE_TEXT, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
Expand Down Expand Up @@ -186,13 +198,77 @@ public void execute() {
settings = Settings.from();
}

if (repositoryName == null) {
repositoryName = ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME;
version = pullFromCentral(settings, orgName, packageName, version);
} else if (!LOCAL_REPOSITORY_NAME.equals(repositoryName)) {
pullFromMavenRepo(settings, orgName, packageName, version);
}

if (!resolveDependencies(orgName, packageName, version)) {
CommandUtil.exitError(this.exitWhenFinish);
return;
}

if (this.exitWhenFinish) {
Runtime.getRuntime().exit(0);
}

}

private String pullFromCentral(Settings settings, String orgName, String packageName, String version) {
Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
.resolve(ProjectConstants.BALA_DIR_NAME)
.resolve(orgName).resolve(packageName);

if (!version.equals(Names.EMPTY.getValue()) && Files.exists(packagePathInBalaCache.resolve(version))) {
outStream.println("Package already exists.\n");
}
// create directory path in bala cache
try {
createDirectories(packagePathInBalaCache);
} catch (IOException e) {
CommandUtil.exitError(this.exitWhenFinish);
throw createLauncherException(
"unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
}

CommandUtil.setPrintStream(errStream);
String supportedPlatform = Arrays.stream(JvmTarget.values())
.map(JvmTarget::code)
.collect(Collectors.joining(","));
CentralAPIClient client;
try {
client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
initializeProxy(settings.getProxy()), settings.getProxy().username(),
settings.getProxy().password(), getAccessTokenOfCLI(settings),
settings.getCentral().getConnectTimeout(),
settings.getCentral().getReadTimeout(), settings.getCentral().getWriteTimeout(),
settings.getCentral().getCallTimeout(), settings.getCentral().getMaxRetries());
client.pullPackage(orgName, packageName, version, packagePathInBalaCache, supportedPlatform,
RepoUtils.getBallerinaVersion(), false);
if (version.equals(Names.EMPTY.getValue())) {
List<String> versions = client.getPackageVersions(orgName, packageName, supportedPlatform,
RepoUtils.getBallerinaVersion());
version = CommandUtil.getLatestVersion(versions);
}
} catch (PackageAlreadyExistsException e) {
outStream.println("Package already exists.\n");
version = e.version();
} catch (CentralClientException e) {
errStream.println("package not found: " + orgName + "/" + packageName);
CommandUtil.exitError(this.exitWhenFinish);
}
return version;
}

private void pullFromMavenRepo(Settings settings, String orgName, String packageName, String version) {
Repository targetRepository = null;
if (repositoryName != null) {
for (Repository repository : settings.getRepositories()) {
if (repositoryName.equals(repository.id())) {
targetRepository = repository;
break;
}
for (Repository repository : settings.getRepositories()) {
if (repositoryName.equals(repository.id())) {
targetRepository = repository;
break;
}
}

Expand All @@ -218,7 +294,8 @@ public void execute() {
Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR)
.resolve(targetRepository.id())
.resolve(ProjectConstants.BALA_DIR_NAME);
.resolve(ProjectConstants.BALA_DIR_NAME)
.resolve(orgName).resolve(packageName).resolve(version);

try {
Path tmpDownloadDirectory = Files.createTempDirectory("ballerina-" + System.nanoTime());
Expand All @@ -233,8 +310,7 @@ public void execute() {
try (BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8)) {
JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
String platform = resultObj.get(PLATFORM).getAsString();
Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
.resolve(version).resolve(platform);
Path actualBalaPath = mavenBalaCachePath.resolve(platform);
org.apache.commons.io.FileUtils.copyDirectory(temporaryExtractionPath.toFile(),
actualBalaPath.toFile());
}
Expand All @@ -247,56 +323,24 @@ public void execute() {
}
PrintStream out = System.out;
out.println("Successfully pulled the package from the custom repository.");
return;
}

Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
.resolve(ProjectConstants.BALA_DIR_NAME)
.resolve(orgName).resolve(packageName);
// create directory path in bala cache
try {
createDirectories(packagePathInBalaCache);
} catch (IOException e) {
CommandUtil.exitError(this.exitWhenFinish);
throw createLauncherException(
"unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
}
}

private boolean resolveDependencies(String orgName, String packageName, String version) {
CommandUtil.setPrintStream(errStream);
String supportedPlatform = Arrays.stream(JvmTarget.values())
.map(JvmTarget::code)
.collect(Collectors.joining(","));
try {
CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
initializeProxy(settings.getProxy()), settings.getProxy().username(),
settings.getProxy().password(), getAccessTokenOfCLI(settings),
settings.getCentral().getConnectTimeout(),
settings.getCentral().getReadTimeout(), settings.getCentral().getWriteTimeout(),
settings.getCentral().getCallTimeout(), settings.getCentral().getMaxRetries());
client.pullPackage(orgName, packageName, version, packagePathInBalaCache, supportedPlatform,
RepoUtils.getBallerinaVersion(), false);
if (version.equals(Names.EMPTY.getValue())) {
List<String> versions = client.getPackageVersions(orgName, packageName, supportedPlatform,
RepoUtils.getBallerinaVersion());
version = CommandUtil.getLatestVersion(versions);
}
boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(orgName, packageName, version);
BuildOptions buildOptions = BuildOptions.builder().setSticky(sticky).setOffline(offline).build();
boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(
orgName, packageName, version, buildOptions, repositoryName);
if (hasCompilationErrors) {
CommandUtil.printError(this.errStream, "compilation contains errors", null, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
return false;
}
} catch (PackageAlreadyExistsException e) {
errStream.println(e.getMessage());
CommandUtil.exitError(this.exitWhenFinish);
} catch (CentralClientException e) {
errStream.println("package not found: " + orgName + "/" + packageName);
CommandUtil.exitError(this.exitWhenFinish);
}
if (this.exitWhenFinish) {
Runtime.getRuntime().exit(0);
} catch (ProjectException e) {
CommandUtil.printError(this.errStream,
"error occurred while resolving dependencies, reason: " + e.getMessage(), null, false);
}
return true;
}

@Override
Expand Down
Loading

0 comments on commit aabac46

Please sign in to comment.