Add custom repository support for selective repositories

cli/ballerina-cli/spotbugs-exclude.xml

@@ -17,6 +17,14 @@
   -->
 
 <FindBugsFilter>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
+    </Match>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
+    </Match>
     <Match>
         <Class name="io.ballerina.cli.cmd.BuildCommand"/>
         <Bug pattern="URF_UNREAD_FIELD"/>

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

@@ -18,28 +18,41 @@
 
 package io.ballerina.cli.cmd;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
 import io.ballerina.cli.BLauncherCmd;
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.SemanticVersion;
 import io.ballerina.projects.Settings;
+import io.ballerina.projects.internal.model.Proxy;
+import io.ballerina.projects.internal.model.Repository;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.central.client.CentralAPIClient;
 import org.ballerinalang.central.client.CentralClientConstants;
 import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.ballerinalang.central.client.exceptions.PackageAlreadyExistsException;
+import org.ballerinalang.maven.bala.client.MavenResolverClient;
+import org.ballerinalang.maven.bala.client.MavenResolverClientException;
 import org.ballerinalang.toml.exceptions.SettingsTomlException;
 import org.wso2.ballerinalang.compiler.util.Names;
 import org.wso2.ballerinalang.util.RepoUtils;
 import picocli.CommandLine;
 
+import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Comparator;
 import java.util.List;
 
 import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
+import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
+import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
 import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
 import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
 import static io.ballerina.projects.util.ProjectUtils.validateOrgName;
@@ -71,6 +84,9 @@
     @CommandLine.Option(names = "--debug", hidden = true)
     private String debugPort;
 
+    @CommandLine.Option(names = "--repository")
+    private String repositoryName;
+
     public PullCommand() {
         this.errStream = System.err;
         this.exitWhenFinish = true;
@@ -163,6 +179,87 @@
             }
         }
 
+        Settings settings;
+        try {
+            settings = RepoUtils.readSettings();
+        } catch (SettingsTomlException e) {
+            settings = Settings.from();
+        }
+
+        Repository targetRepository = null;
+        if (repositoryName != null) {
+            for (Repository repository : settings.getRepositories()) {
+                if (repository.id().equals(repositoryName)) {
+                    targetRepository = repository;
+                    break;
+                }
+            }
+        }
+
+        if (targetRepository == null && repositoryName != null) {
+            String errMsg = "unsupported repository '" + repositoryName + "' found. Only " +
+                    "repositories mentioned in the Settings.toml are supported.";
+            CommandUtil.printError(this.errStream, errMsg, null, false);
+            CommandUtil.exitError(this.exitWhenFinish);
+            return;
+        }
+
+        if (targetRepository != null) {
+            MavenResolverClient mavenResolverClient = new MavenResolverClient();
+            if (!"".equals(targetRepository.username()) && !"".equals(targetRepository.password())) {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url(),
+                        targetRepository.username(), targetRepository.password());
+            } else {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url());
+            }
+            Proxy proxy = settings.getProxy();
+            mavenResolverClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());
+
+            Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
+                    .resolve(ProjectConstants.REPOSITORIES_DIR)
+                    .resolve(targetRepository.id())
+                    .resolve(ProjectConstants.BALA_DIR_NAME);
+
+            try {
+                mavenResolverClient.pullPackage(orgName, packageName, version,
+                        String.valueOf(mavenBalaCachePath.toAbsolutePath()));
+                Path balaDownloadPath = mavenBalaCachePath.resolve(orgName).resolve(packageName).resolve(version)
+                        .resolve(packageName + "-" + version + BALA_EXTENSION);
+                Path balaHashPath = mavenBalaCachePath.resolve(orgName).resolve(packageName).resolve(version)
+                        .resolve(packageName + "-" + version + BALA_EXTENSION + ".sha1");
+                Path temporaryExtractionPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
+                        .resolve(version).resolve(PLATFORM);
+                ProjectUtils.extractBala(balaDownloadPath, temporaryExtractionPath);
+                Path packageJsonPath = temporaryExtractionPath.resolve("package.json");
+                BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8);
+                JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
+                String platform = resultObj.get(PLATFORM).getAsString();
+                Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
+                        .resolve(version).resolve(platform);
+                temporaryExtractionPath.toFile().renameTo(actualBalaPath.toFile());
+
+                Files.delete(balaDownloadPath);
+                Files.delete(balaHashPath);
+                Files.delete(balaDownloadPath.getParent().resolve("_remote.repositories"));
+                if (Files.exists(temporaryExtractionPath)) {
+                    Files.walk(temporaryExtractionPath)
+                            .sorted(Comparator.reverseOrder())
+                            .map(Path::toFile)
+                            .forEach(File::delete);
+                }
+
+            } catch (MavenResolverClientException e) {
+                errStream.println("unexpected error occurred while pulling package:" + e.getMessage());
+                CommandUtil.exitError(this.exitWhenFinish);
+            } catch (IOException e) {
+                throw createLauncherException(
+                        "unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
+            }
+            PrintStream out = System.out;
+            out.println("Successfully pulled the package from the custom repository.");
+            return;
+        }
+
         Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
                 .resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
                 .resolve(ProjectConstants.BALA_DIR_NAME)
@@ -180,14 +277,6 @@
         for (int i = 0; i < SUPPORTED_PLATFORMS.length; i++) {
             String supportedPlatform = SUPPORTED_PLATFORMS[i];
             try {
-                Settings settings;
-                try {
-                    settings = RepoUtils.readSettings();
-                    // Ignore Settings.toml diagnostics in the pull command
-                } catch (SettingsTomlException e) {
-                    // Ignore 'Settings.toml' parsing errors and return empty Settings object
-                    settings = Settings.from();
-                }
                 CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
                         initializeProxy(settings.getProxy()), settings.getProxy().username(),
                         settings.getProxy().password(), getAccessTokenOfCLI(settings));

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

@@ -29,13 +29,17 @@
 import io.ballerina.projects.Settings;
 import io.ballerina.projects.bala.BalaProject;
 import io.ballerina.projects.directory.BuildProject;
+import io.ballerina.projects.internal.model.Proxy;
+import io.ballerina.projects.internal.model.Repository;
 import io.ballerina.projects.repos.TempDirCompilationCache;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.central.client.CentralAPIClient;
 import org.ballerinalang.central.client.CentralClientConstants;
 import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.ballerinalang.central.client.exceptions.NoPackageException;
+import org.ballerinalang.maven.bala.client.MavenResolverClient;
+import org.ballerinalang.maven.bala.client.MavenResolverClientException;
 import org.ballerinalang.toml.exceptions.SettingsTomlException;
 import org.wso2.ballerinalang.util.RepoUtils;
 import picocli.CommandLine;
@@ -48,6 +52,7 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
@@ -61,6 +66,7 @@
 import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
 import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
 import static io.ballerina.runtime.api.constants.RuntimeConstants.SYSTEM_PROP_BAL_DEBUG;
+import static java.nio.file.Files.createDirectories;
 import static org.wso2.ballerinalang.programfile.ProgramFileConstants.SUPPORTED_PLATFORMS;
 
 /**
@@ -145,19 +151,33 @@
         System.setProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM, "true");
 
         try {
+            Settings settings = RepoUtils.readSettings();
+
             // If the repository flag is specified, validate and push to the provided repo
             if (repositoryName != null) {
-                if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
+                boolean isCustomRepository = false;
+                Repository targetRepository = null;
+                for (Repository repository : settings.getRepositories()) {
+                    if (repository.id().equals(repositoryName)) {
+                        isCustomRepository = true;
+                        targetRepository = repository;
+                        break;
+                    }
+                }
+
+                if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME) && !isCustomRepository) {
                     String errMsg = "unsupported repository '" + repositoryName + "' found. Only '"
-                            + ProjectConstants.LOCAL_REPOSITORY_NAME + "' repository is supported.";
+                            + ProjectConstants.LOCAL_REPOSITORY_NAME +
+                            "' repository and repositories mentioned in the Settings.toml are supported.";
                     CommandUtil.printError(this.errStream, errMsg, null, false);
                     CommandUtil.exitError(this.exitWhenFinish);
                     return;
                 }
 
-                if (balaPath == null) {
+                if (balaPath == null && repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
                     pushPackage(project);
-                } else {
+                    return;
+                } else if (repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
                     if (!balaPath.toFile().exists()) {
                         throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
                     }
@@ -166,9 +186,41 @@
                     }
                     validatePackageMdAndBalToml(balaPath);
                     pushBalaToCustomRepo(balaPath);
+                    return;
+                }
+
+                MavenResolverClient mvnClient = new MavenResolverClient();
+                if (!"".equals(targetRepository.username()) && !"".equals(targetRepository.password())) {
+                    mvnClient.addRepository(targetRepository.id(), targetRepository.url(), targetRepository.username(),
+                                            targetRepository.password());
+                } else {
+                    mvnClient.addRepository(targetRepository.id(), targetRepository.url());
                 }
+                Proxy proxy = settings.getProxy();
+                mvnClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());
+
+                Path customRepoLocalCache = RepoUtils.createAndGetHomeReposPath()
+                        .resolve(ProjectConstants.REPOSITORIES_DIR)
+                        .resolve(targetRepository.id())
+                        .resolve(ProjectConstants.BALA_DIR_NAME)
+                        .resolve("temp");
+
+
+                if (balaPath == null && isCustomRepository) {
+                    pushPackage(project, mvnClient, customRepoLocalCache);
+                } else if (isCustomRepository) {
+                    if (!balaPath.toFile().exists()) {
+                        throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
+                    }
+                    if (!FileUtils.getExtension(balaPath).equals("bala")) {
+                        throw new ProjectException("file provided is not a bala file: " + balaPath + ".");
+                    }
+                    validatePackageMdAndBalToml(balaPath);
+                    pushBalaToCustomRepo(balaPath, mvnClient, customRepoLocalCache);
+                }
+
+
             } else {
-                Settings settings = RepoUtils.readSettings();
                 if (settings.diagnostics().hasErrors()) {
                     CommandUtil.printError(this.errStream, settings.getErrorMessage(), null, false);
                     CommandUtil.exitError(this.exitWhenFinish);
@@ -226,6 +278,11 @@
         pushBalaToCustomRepo(balaFilePath);
     }
 
+    private void pushPackage(BuildProject project, MavenResolverClient client, Path customRepoPath) {
+        Path balaFilePath = validateBalaFile(project, this.balaPath);
+        pushBalaToCustomRepo(balaFilePath, client, customRepoPath);
+    }
+
     private void pushPackage(BuildProject project, CentralAPIClient client)
             throws CentralClientException {
         Path balaFilePath = validateBala(project, client, this.balaPath);
@@ -402,6 +459,45 @@
         }
     }
 
+    private void pushBalaToCustomRepo(Path balaPath, MavenResolverClient client, Path customRepoPath) {
+        try {
+            createDirectories(customRepoPath);
+        } catch (IOException e) {
+            throw new ProjectException(e.getMessage());
+        }
+        Path balaFileName = balaPath.getFileName();
+        if (null != balaFileName) {
+            ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
+            defaultBuilder.addCompilationCacheFactory(TempDirCompilationCache::from);
+            BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);
+
+            String org = balaProject.currentPackage().manifest().org().toString();
+            String name = balaProject.currentPackage().manifest().name().toString();
+            String version = balaProject.currentPackage().manifest().version().toString();
+
+            try {
+                client.pushPackage(balaPath, org, name, version, customRepoPath);
+                if (Files.exists(customRepoPath)) {
+                    Files.walk(customRepoPath)
+                            .sorted(Comparator.reverseOrder())
+                            .map(Path::toFile)
+                            .forEach(File::delete);
+                }
+            } catch (MavenResolverClientException | IOException e) {
+                throw new ProjectException(e.getMessage());
+            }
+
+            Path relativePathToBalaFile;
+            if (this.balaPath != null) {
+                relativePathToBalaFile = balaPath;
+            } else {
+                relativePathToBalaFile = userDir.relativize(balaPath);
+            }
+            outStream.println("Successfully pushed " + relativePathToBalaFile
+                    + " to '" + repositoryName + "' repository.");
+        }
+    }
+
     /**
      * Check if package already available in the remote.
      *