Sync with master

bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/TypeChecker.java

@@ -1152,6 +1152,12 @@ private static boolean checkIsRecordType(BRecordType sourceRecordType, BRecordTy
                 if (!SymbolFlags.isFlagOn(targetField.getFlags(), SymbolFlags.OPTIONAL)) {
                     return false;
                 }
+
+                if (!sourceRecordType.sealed && !checkIsType(sourceRecordType.restFieldType, targetField.getFieldType(),
+                                                             unresolvedTypes)) {
+                    return false;
+                }
+
                 continue;
             }
 
@@ -1313,6 +1319,12 @@ private static boolean checkIsRecordType(MapValue sourceRecordValue, BRecordType
                 if (!SymbolFlags.isFlagOn(targetField.getFlags(), SymbolFlags.OPTIONAL)) {
                     return false;
                 }
+
+                if (!sourceRecordType.sealed && !checkIsType(sourceRecordType.restFieldType, targetField.getFieldType(),
+                                                             unresolvedTypes)) {
+                    return false;
+                }
+
                 continue;
             }
 

bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/util/LargeStructureUtils.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved.
+ *
+ * WSO2 LLC. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.ballerina.runtime.internal.util;
+
+import io.ballerina.runtime.api.values.BArray;
+import io.ballerina.runtime.api.values.BMap;
+import io.ballerina.runtime.api.values.BMapInitialValueEntry;
+import io.ballerina.runtime.internal.values.HandleValue;
+import io.ballerina.runtime.internal.values.ListInitialValueEntry;
+import io.ballerina.runtime.internal.values.MappingInitialValueEntry;
+
+/**
+ * Util methods required for handling large arrays, tuples, maps and records.
+ *
+ * @since 2201.8.0
+ */
+public class LargeStructureUtils {
+
+    private LargeStructureUtils() {}
+
+    public static HandleValue getListInitialValueEntryArray(long size) {
+        return new HandleValue(new ListInitialValueEntry[(int) size]);
+    }
+
+    public static void setExpressionEntry(HandleValue arrayList, Object element, long index) {
+        ListInitialValueEntry[] arr = (ListInitialValueEntry[]) arrayList.getValue();
+        arr[(int) index] = new ListInitialValueEntry.ExpressionEntry(element);
+    }
+
+    public static void setSpreadEntry(HandleValue arrayList, Object element, long index) {
+        ListInitialValueEntry[] arr = (ListInitialValueEntry[]) arrayList.getValue();
+        arr[(int) index] = new ListInitialValueEntry.SpreadEntry((BArray) element);
+    }
+
+    public static HandleValue getBMapInitialValueEntryArray(long size) {
+        return new HandleValue(new BMapInitialValueEntry[(int) size]);
+    }
+
+    public static void setKeyValueEntry(HandleValue arrayList, Object key, Object value, long index) {
+        BMapInitialValueEntry[] arr = (BMapInitialValueEntry[]) arrayList.getValue();
+        arr[(int) index] = new MappingInitialValueEntry.KeyValueEntry(key, value);
+    }
+
+    public static void setSpreadFieldEntry(HandleValue arrayList, Object spreadFieldEntry, long index) {
+        BMapInitialValueEntry[] arr = (BMapInitialValueEntry[]) arrayList.getValue();
+        arr[(int) index] = new MappingInitialValueEntry.SpreadFieldEntry((BMap) spreadFieldEntry);
+    }
+
+}

bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/values/HandleValue.java

@@ -40,7 +40,6 @@ public class HandleValue implements BHandle, RefValue {
     private Object value;
     private BTypedesc typedesc;
 
-    @Deprecated
     public HandleValue(Object value) {
         this.value = value;
     }

cli/ballerina-cli/build.gradle

@@ -58,6 +58,7 @@ dependencies {
     implementation group: 'org.ow2.asm', name: 'asm', version: "${project.ow2AsmVersion}"
     implementation group: 'org.ow2.asm', name: 'asm-commons', version: "${project.ow2AsmCommonsVersion}"
     implementation group: 'org.ow2.asm', name: 'asm-tree', version: "${project.ow2AsmTreeVersion}"
+    implementation 'commons-io:commons-io'
 
     testImplementation "org.testng:testng:${project.testngVersion}"
     testImplementation "org.mockito:mockito-core:${project.mockitoCoreVersion}"

cli/ballerina-cli/spotbugs-exclude.xml

@@ -17,6 +17,14 @@
   -->
 
 <FindBugsFilter>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
+    </Match>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
+    </Match>
     <Match>
         <Class name="io.ballerina.cli.cmd.BuildCommand"/>
         <Bug pattern="URF_UNREAD_FIELD"/>

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

@@ -18,28 +18,39 @@
 
 package io.ballerina.cli.cmd;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
 import io.ballerina.cli.BLauncherCmd;
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.SemanticVersion;
 import io.ballerina.projects.Settings;
+import io.ballerina.projects.internal.model.Proxy;
+import io.ballerina.projects.internal.model.Repository;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.central.client.CentralAPIClient;
 import org.ballerinalang.central.client.CentralClientConstants;
 import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.ballerinalang.central.client.exceptions.PackageAlreadyExistsException;
+import org.ballerinalang.maven.bala.client.MavenResolverClient;
+import org.ballerinalang.maven.bala.client.MavenResolverClientException;
 import org.ballerinalang.toml.exceptions.SettingsTomlException;
 import org.wso2.ballerinalang.compiler.util.Names;
 import org.wso2.ballerinalang.util.RepoUtils;
 import picocli.CommandLine;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 
 import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
+import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
+import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
 import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
 import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
 import static io.ballerina.projects.util.ProjectUtils.validateOrgName;
@@ -71,6 +82,9 @@ public class PullCommand implements BLauncherCmd {
     @CommandLine.Option(names = "--debug", hidden = true)
     private String debugPort;
 
+    @CommandLine.Option(names = "--repository")
+    private String repositoryName;
+
     public PullCommand() {
         this.errStream = System.err;
         this.exitWhenFinish = true;
@@ -163,6 +177,77 @@ public void execute() {
             }
         }
 
+        Settings settings;
+        try {
+            settings = RepoUtils.readSettings();
+        } catch (SettingsTomlException e) {
+            settings = Settings.from();
+        }
+
+        Repository targetRepository = null;
+        if (repositoryName != null) {
+            for (Repository repository : settings.getRepositories()) {
+                if (repositoryName.equals(repository.id())) {
+                    targetRepository = repository;
+                    break;
+                }
+            }
+        }
+
+        if (targetRepository == null && repositoryName != null) {
+            String errMsg = "unsupported repository '" + repositoryName + "' found. Only " +
+                    "repositories mentioned in the Settings.toml are supported.";
+            CommandUtil.printError(this.errStream, errMsg, null, false);
+            CommandUtil.exitError(this.exitWhenFinish);
+            return;
+        }
+
+        if (targetRepository != null) {
+            MavenResolverClient mavenResolverClient = new MavenResolverClient();
+            if (!targetRepository.username().isEmpty() && !targetRepository.password().isEmpty()) {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url(),
+                        targetRepository.username(), targetRepository.password());
+            } else {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url());
+            }
+            Proxy proxy = settings.getProxy();
+            mavenResolverClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());
+
+            Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
+                    .resolve(ProjectConstants.REPOSITORIES_DIR)
+                    .resolve(targetRepository.id())
+                    .resolve(ProjectConstants.BALA_DIR_NAME);
+
+            try {
+                Path tmpDownloadDirectory = Files.createTempDirectory("ballerina-" + System.nanoTime());
+                mavenResolverClient.pullPackage(orgName, packageName, version,
+                        String.valueOf(tmpDownloadDirectory.toAbsolutePath()));
+                Path balaDownloadPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName).resolve(version)
+                        .resolve(packageName + "-" + version + BALA_EXTENSION);
+                Path temporaryExtractionPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName)
+                        .resolve(version).resolve(PLATFORM);
+                ProjectUtils.extractBala(balaDownloadPath, temporaryExtractionPath);
+                Path packageJsonPath = temporaryExtractionPath.resolve("package.json");
+                try (BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8)) {
+                    JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
+                    String platform = resultObj.get(PLATFORM).getAsString();
+                    Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
+                            .resolve(version).resolve(platform);
+                    org.apache.commons.io.FileUtils.copyDirectory(temporaryExtractionPath.toFile(),
+                            actualBalaPath.toFile());
+                }
+            } catch (MavenResolverClientException e) {
+                errStream.println("unexpected error occurred while pulling package:" + e.getMessage());
+                CommandUtil.exitError(this.exitWhenFinish);
+            } catch (IOException e) {
+                throw createLauncherException(
+                        "unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
+            }
+            PrintStream out = System.out;
+            out.println("Successfully pulled the package from the custom repository.");
+            return;
+        }
+
         Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
                 .resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
                 .resolve(ProjectConstants.BALA_DIR_NAME)
@@ -180,14 +265,6 @@ public void execute() {
         for (int i = 0; i < SUPPORTED_PLATFORMS.length; i++) {
             String supportedPlatform = SUPPORTED_PLATFORMS[i];
             try {
-                Settings settings;
-                try {
-                    settings = RepoUtils.readSettings();
-                    // Ignore Settings.toml diagnostics in the pull command
-                } catch (SettingsTomlException e) {
-                    // Ignore 'Settings.toml' parsing errors and return empty Settings object
-                    settings = Settings.from();
-                }
                 CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
                         initializeProxy(settings.getProxy()), settings.getProxy().username(),
                         settings.getProxy().password(), getAccessTokenOfCLI(settings));