Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add custom repository support for selective repositories #41177

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cli/ballerina-cli/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ dependencies {
implementation group: 'org.ow2.asm', name: 'asm', version: "${project.ow2AsmVersion}"
implementation group: 'org.ow2.asm', name: 'asm-commons', version: "${project.ow2AsmCommonsVersion}"
implementation group: 'org.ow2.asm', name: 'asm-tree', version: "${project.ow2AsmTreeVersion}"
implementation 'commons-io:commons-io'

testImplementation "org.testng:testng:${project.testngVersion}"
testImplementation "org.mockito:mockito-core:${project.mockitoCoreVersion}"
Expand Down
8 changes: 8 additions & 0 deletions cli/ballerina-cli/spotbugs-exclude.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@
-->

<FindBugsFilter>
<Match>
<Class name="io.ballerina.cli.cmd.PullCommand"/>
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
</Match>
<Match>
<Class name="io.ballerina.cli.cmd.PullCommand"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
</Match>
<Match>
<Class name="io.ballerina.cli.cmd.BuildCommand"/>
<Bug pattern="URF_UNREAD_FIELD"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,39 @@

package io.ballerina.cli.cmd;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.ballerina.cli.BLauncherCmd;
import io.ballerina.projects.ProjectException;
import io.ballerina.projects.SemanticVersion;
import io.ballerina.projects.Settings;
import io.ballerina.projects.internal.model.Proxy;
import io.ballerina.projects.internal.model.Repository;
import io.ballerina.projects.util.ProjectConstants;
import io.ballerina.projects.util.ProjectUtils;
import org.ballerinalang.central.client.CentralAPIClient;
import org.ballerinalang.central.client.CentralClientConstants;
import org.ballerinalang.central.client.exceptions.CentralClientException;
import org.ballerinalang.central.client.exceptions.PackageAlreadyExistsException;
import org.ballerinalang.maven.bala.client.MavenResolverClient;
import org.ballerinalang.maven.bala.client.MavenResolverClientException;
import org.ballerinalang.toml.exceptions.SettingsTomlException;
import org.wso2.ballerinalang.compiler.util.Names;
import org.wso2.ballerinalang.util.RepoUtils;
import picocli.CommandLine;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;

import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
import static io.ballerina.projects.util.ProjectUtils.validateOrgName;
Expand Down Expand Up @@ -71,6 +82,9 @@
@CommandLine.Option(names = "--debug", hidden = true)
private String debugPort;

@CommandLine.Option(names = "--repository")
private String repositoryName;

public PullCommand() {
this.errStream = System.err;
this.exitWhenFinish = true;
Expand Down Expand Up @@ -163,6 +177,77 @@
}
}

Settings settings;
try {
settings = RepoUtils.readSettings();
} catch (SettingsTomlException e) {
settings = Settings.from();

Check warning on line 184 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java#L183-L184

Added lines #L183 - L184 were not covered by tests
}

Repository targetRepository = null;
if (repositoryName != null) {
for (Repository repository : settings.getRepositories()) {
if (repositoryName.equals(repository.id())) {
targetRepository = repository;
break;
}
}
}

if (targetRepository == null && repositoryName != null) {
String errMsg = "unsupported repository '" + repositoryName + "' found. Only " +
"repositories mentioned in the Settings.toml are supported.";
CommandUtil.printError(this.errStream, errMsg, null, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
}

if (targetRepository != null) {
MavenResolverClient mavenResolverClient = new MavenResolverClient();
if (!targetRepository.username().isEmpty() && !targetRepository.password().isEmpty()) {
mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url(),
targetRepository.username(), targetRepository.password());

Check warning on line 209 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java#L208-L209

Added lines #L208 - L209 were not covered by tests
} else {
mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url());
}
Proxy proxy = settings.getProxy();
mavenResolverClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());

Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR)
.resolve(targetRepository.id())
.resolve(ProjectConstants.BALA_DIR_NAME);

try {
Path tmpDownloadDirectory = Files.createTempDirectory("ballerina-" + System.nanoTime());
mavenResolverClient.pullPackage(orgName, packageName, version,
String.valueOf(tmpDownloadDirectory.toAbsolutePath()));
Path balaDownloadPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName).resolve(version)
.resolve(packageName + "-" + version + BALA_EXTENSION);
Path temporaryExtractionPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName)
.resolve(version).resolve(PLATFORM);
ProjectUtils.extractBala(balaDownloadPath, temporaryExtractionPath);
Path packageJsonPath = temporaryExtractionPath.resolve("package.json");
try (BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8)) {
JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
String platform = resultObj.get(PLATFORM).getAsString();
Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
.resolve(version).resolve(platform);
org.apache.commons.io.FileUtils.copyDirectory(temporaryExtractionPath.toFile(),
actualBalaPath.toFile());
}
} catch (MavenResolverClientException e) {
errStream.println("unexpected error occurred while pulling package:" + e.getMessage());
CommandUtil.exitError(this.exitWhenFinish);
} catch (IOException e) {
throw createLauncherException(
"unexpected error occurred while creating package repository in bala cache: " + e.getMessage());

Check warning on line 244 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java#L239-L244

Added lines #L239 - L244 were not covered by tests
}
PrintStream out = System.out;
out.println("Successfully pulled the package from the custom repository.");
return;
}

Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
.resolve(ProjectConstants.BALA_DIR_NAME)
Expand All @@ -180,14 +265,6 @@
for (int i = 0; i < SUPPORTED_PLATFORMS.length; i++) {
String supportedPlatform = SUPPORTED_PLATFORMS[i];
try {
Settings settings;
try {
settings = RepoUtils.readSettings();
// Ignore Settings.toml diagnostics in the pull command
} catch (SettingsTomlException e) {
// Ignore 'Settings.toml' parsing errors and return empty Settings object
settings = Settings.from();
}
CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
initializeProxy(settings.getProxy()), settings.getProxy().username(),
settings.getProxy().password(), getAccessTokenOfCLI(settings));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,17 @@
import io.ballerina.projects.Settings;
import io.ballerina.projects.bala.BalaProject;
import io.ballerina.projects.directory.BuildProject;
import io.ballerina.projects.internal.model.Proxy;
import io.ballerina.projects.internal.model.Repository;
import io.ballerina.projects.repos.TempDirCompilationCache;
import io.ballerina.projects.util.ProjectConstants;
import io.ballerina.projects.util.ProjectUtils;
import org.ballerinalang.central.client.CentralAPIClient;
import org.ballerinalang.central.client.CentralClientConstants;
import org.ballerinalang.central.client.exceptions.CentralClientException;
import org.ballerinalang.central.client.exceptions.NoPackageException;
import org.ballerinalang.maven.bala.client.MavenResolverClient;
import org.ballerinalang.maven.bala.client.MavenResolverClientException;
import org.ballerinalang.toml.exceptions.SettingsTomlException;
import org.wso2.ballerinalang.util.RepoUtils;
import picocli.CommandLine;
Expand Down Expand Up @@ -145,19 +149,33 @@
System.setProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM, "true");

try {
Settings settings = RepoUtils.readSettings();

Check warning on line 152 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L152

Added line #L152 was not covered by tests

// If the repository flag is specified, validate and push to the provided repo
if (repositoryName != null) {
if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
boolean isCustomRepository = false;
Repository targetRepository = null;

Check warning on line 157 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L156-L157

Added lines #L156 - L157 were not covered by tests
for (Repository repository : settings.getRepositories()) {
if (repositoryName.equals(repository.id())) {
isCustomRepository = true;
targetRepository = repository;
break;

Check warning on line 162 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L160-L162

Added lines #L160 - L162 were not covered by tests
}
}

if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME) && !isCustomRepository) {
String errMsg = "unsupported repository '" + repositoryName + "' found. Only '"
+ ProjectConstants.LOCAL_REPOSITORY_NAME + "' repository is supported.";
+ ProjectConstants.LOCAL_REPOSITORY_NAME +
"' repository and repositories mentioned in the Settings.toml are supported.";
CommandUtil.printError(this.errStream, errMsg, null, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
}

if (balaPath == null) {
if (balaPath == null && repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
pushPackage(project);
} else {
return;

Check warning on line 177 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L177

Added line #L177 was not covered by tests
} else if (repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
if (!balaPath.toFile().exists()) {
throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
}
Expand All @@ -166,9 +184,34 @@
}
validatePackageMdAndBalToml(balaPath);
pushBalaToCustomRepo(balaPath);
return;

Check warning on line 187 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L187

Added line #L187 was not covered by tests
}

MavenResolverClient mvnClient = new MavenResolverClient();

Check warning on line 190 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L190

Added line #L190 was not covered by tests
if (!targetRepository.username().isEmpty() && !targetRepository.password().isEmpty()) {
mvnClient.addRepository(targetRepository.id(), targetRepository.url(), targetRepository.username(),
targetRepository.password());

Check warning on line 193 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L192-L193

Added lines #L192 - L193 were not covered by tests
} else {
mvnClient.addRepository(targetRepository.id(), targetRepository.url());

Check warning on line 195 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L195

Added line #L195 was not covered by tests
}
Proxy proxy = settings.getProxy();
mvnClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());

Check warning on line 198 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L197-L198

Added lines #L197 - L198 were not covered by tests

if (balaPath == null && isCustomRepository) {
pushPackage(project, mvnClient);

Check warning on line 201 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L201

Added line #L201 was not covered by tests
} else if (isCustomRepository) {
if (!balaPath.toFile().exists()) {
throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");

Check warning on line 204 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L204

Added line #L204 was not covered by tests
}
if (!FileUtils.getExtension(balaPath).equals("bala")) {
throw new ProjectException("file provided is not a bala file: " + balaPath + ".");

Check warning on line 207 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L207

Added line #L207 was not covered by tests
}
validatePackageMdAndBalToml(balaPath);
pushBalaToCustomRepo(balaPath, mvnClient);

Check warning on line 210 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L209-L210

Added lines #L209 - L210 were not covered by tests
}


} else {
Settings settings = RepoUtils.readSettings();
if (settings.diagnostics().hasErrors()) {
CommandUtil.printError(this.errStream, settings.getErrorMessage(), null, false);
CommandUtil.exitError(this.exitWhenFinish);
Expand Down Expand Up @@ -226,6 +269,11 @@
pushBalaToCustomRepo(balaFilePath);
}

private void pushPackage(BuildProject project, MavenResolverClient client) {
Path balaFilePath = validateBalaFile(project, this.balaPath);
pushBalaToCustomRepo(balaFilePath, client);
}

Check warning on line 275 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L273-L275

Added lines #L273 - L275 were not covered by tests

private void pushPackage(BuildProject project, CentralAPIClient client)
throws CentralClientException {
Path balaFilePath = validateBala(project, client, this.balaPath);
Expand Down Expand Up @@ -402,6 +450,35 @@
}
}

private void pushBalaToCustomRepo(Path balaPath, MavenResolverClient client) {
Path balaFileName = balaPath.getFileName();

Check warning on line 454 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L454

Added line #L454 was not covered by tests
if (null != balaFileName) {
ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
defaultBuilder.addCompilationCacheFactory(TempDirCompilationCache::from);
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);

Check warning on line 458 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L456-L458

Added lines #L456 - L458 were not covered by tests

String org = balaProject.currentPackage().manifest().org().toString();
String name = balaProject.currentPackage().manifest().name().toString();
String version = balaProject.currentPackage().manifest().version().toString();

Check warning on line 462 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L460-L462

Added lines #L460 - L462 were not covered by tests

try {
Path customRepoPath = Files.createTempDirectory("ballerina-" + System.nanoTime());
client.pushPackage(balaPath, org, name, version, customRepoPath);
} catch (MavenResolverClientException | IOException e) {
throw new ProjectException(e.getMessage());
}

Check warning on line 469 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L465-L469

Added lines #L465 - L469 were not covered by tests

Path relativePathToBalaFile;
if (this.balaPath != null) {
relativePathToBalaFile = balaPath;

Check warning on line 473 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L473

Added line #L473 was not covered by tests
} else {
relativePathToBalaFile = userDir.relativize(balaPath);

Check warning on line 475 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L475

Added line #L475 was not covered by tests
}
outStream.println("Successfully pushed " + relativePathToBalaFile

Check warning on line 477 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L477

Added line #L477 was not covered by tests
+ " to '" + repositoryName + "' repository.");
}
}

Check warning on line 480 in cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

View check run for this annotation

Codecov / codecov/patch

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java#L480

Added line #L480 was not covered by tests

/**
* Check if package already available in the remote.
*
Expand Down
1 change: 1 addition & 0 deletions cli/ballerina-cli/src/main/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@
requires io.ballerina.toml;
requires io.ballerina.identifier;
requires org.objectweb.asm;
requires org.apache.commons.io;
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
NAME
ballerina-pull - Fetch packages from Ballerina Central
ballerina-pull - Fetch packages from Ballerina Central or a custom
package repository

SYNOPSIS
bal pull <org-name>/<package-name>[:<version>]
Expand All @@ -15,6 +16,14 @@ DESCRIPTION
Organizations are unique within a repository and can be mapped to an
individual user or organization registered with the repository.

To download a package from a custom repository, configure it in the Settings.toml
file and pass the given id with the --repository flag.

OPTIONS
--repository
Pull a package from a custom repository.
The repository must be configured in the <USER_HOME>/.ballerina/Settings.toml file.


EXAMPLES
Pull the latest version of the 'gmail' connector in the 'wso2'
Expand All @@ -24,3 +33,7 @@ EXAMPLES
Pull the '1.1.0' version of the 'gmail' connector in the 'wso2'
organization from Ballerina Central.
$ bal pull wso2/gmail:1.1.0

Pull the '1.1.0' version of the 'gmail' connector in the 'wso2'
organization from the github package repository defined in the Settings.toml file.
$ bal pull wso2/gmail:1.1.0 --repository=wso2
Original file line number Diff line number Diff line change
@@ -1,24 +1,27 @@
NAME
ballerina-push - Push the Ballerina Archive (BALA) of the current package
to Ballerina Central or to the local repository
to a package repository

SYNOPSIS
bal push [OPTIONS] <bala-path>


DESCRIPTION
Push the Ballerina archive (.bala) of the current package to Ballerina
Central or to a local repository. Once the package is pushed to Ballerina
Central, local or a custom remote repository. Once the package is pushed to Ballerina
Central, it becomes public and sharable and will be permanent.

To be able to publish a package to Ballerina Central, you should sign in
to Ballerina Central and obtain an access token.

To be able to publish a package to a custom remote repository, it must be defined
in the <USER_HOME>/.ballerina/Settings.toml file.


OPTIONS
--repository
Push the BALA of the current package to a custom repository.
Only 'local' is allowed.
Only 'local' and repositories specified in the Settings.toml are allowed.

EXAMPLES
Push the BALA of the current package to Ballerina Central.
Expand Down
Loading
Loading