diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java b/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java index f3e6ac018a..c0ceb425d7 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java @@ -14,6 +14,7 @@ import java.io.StringWriter; import java.nio.charset.Charset; import java.util.AbstractMap; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -25,9 +26,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; @@ -113,6 +114,7 @@ public class CycloneDxBOMImporter { private static final String INVALID_COMPONENT = "invalidComp"; private static final String INVALID_RELEASE = "invalidRel"; private static final String INVALID_PACKAGE = "invalidPkg"; + private static final String INVALID_VCS = "invalidVCS"; private static final String PROJECT_ID = "projectId"; private static final String PROJECT_NAME = "projectName"; private static final boolean IS_PACKAGE_PORTLET_ENABLED = SW360Constants.IS_PACKAGE_PORTLET_ENABLED; @@ -142,13 +144,11 @@ public CycloneDxBOMImporter(ProjectDatabaseHandler projectDatabaseHandler, Compo * @return Map> */ private Map> getVcsToComponentMap(List components) { - return components.stream().filter(Objects::nonNull) + return components.parallelStream().filter(Objects::nonNull) .flatMap(comp -> CommonUtils.nullToEmptyList(comp.getExternalReferences()).stream() .filter(Objects::nonNull) .filter(ref -> ExternalReference.Type.VCS.equals(ref.getType())) .map(ExternalReference::getUrl) - .map(String::toLowerCase) - .map(url -> StringUtils.removeEnd(url, DOT_GIT)) .map(url -> new AbstractMap.SimpleEntry<>(url, comp))) .collect(Collectors.groupingBy(e -> e.getKey(), Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); @@ -190,21 +190,15 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a .filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count(); long componentsCount = components.size(); org.cyclonedx.model.Component compMetadata = bomMetadata.getComponent(); - Map> vcsToComponentMap = new HashMap<>(); + final Map> vcsToComponentMap = new HashMap<>(); if (!IS_PACKAGE_PORTLET_ENABLED) { vcsToComponentMap.put("", components); requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent); } else { + vcsToComponentMap.putAll(getVcsToComponentMap(components)); - vcsToComponentMap = getVcsToComponentMap(components); - if (componentsCount == vcsCount) { - - requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent); - } else if (componentsCount > vcsCount) { - - requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent); - + requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent); if (requestSummary.requestStatus.equals(RequestStatus.SUCCESS)) { String jsonMessage = requestSummary.getMessage(); @@ -214,6 +208,14 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a if (CommonUtils.isNullEmptyOrWhitespace(projId)) { return requestSummary; } + final List componentsWithInvalidVCS = new ArrayList<>(); + + if (CommonUtils.isNotNullEmptyOrWhitespace(messageMap.get(INVALID_VCS))) { + componentsWithInvalidVCS.addAll(Stream.of(messageMap.get(INVALID_VCS).split("\\|\\|")) + .map(vcs -> vcsToComponentMap.get(vcs)).flatMap(List::stream) + .collect(Collectors.toList())); + } + final Set duplicatePackages = new HashSet<>(); final Set componentsWithoutVcs = new HashSet<>(); final Set invalidPackages = new HashSet<>(); @@ -236,7 +238,8 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a for (org.cyclonedx.model.Component comp : components) { if (CommonUtils.isNullOrEmptyCollection(comp.getExternalReferences()) - || comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0) { + || comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0 + || componentsWithInvalidVCS.contains(comp)) { final var fullName = SW360Utils.getVersionedName(comp.getName(), comp.getVersion()); final var licenses = getLicenseFromBomComponent(comp); @@ -297,13 +300,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a messageMap.put(PKG_REUSE_COUNT_KEY, String.valueOf(pkgReuseCount)); requestSummary.setMessage(convertCollectionToJSONString(messageMap)); } - } else { - requestSummary.setMessage(String.format(String.format( - "SBOM import aborted with error: Multiple vcs information found in compnents, vcs found: %s and total components: %s", - vcsCount, componentsCount))); - return requestSummary; } - } if (RequestStatus.SUCCESS.equals(requestSummary.getRequestStatus())) { String jsonMessage = requestSummary.getMessage(); @@ -553,6 +550,7 @@ private Map importAllComponentsAsPackages(Map(); final Set duplicateComponents = new HashSet<>(); + final Set invalidVCS = new HashSet<>(); final Set duplicateReleases = new HashSet<>(); final Set duplicatePackages = new HashSet<>(); final Set invalidReleases = new HashSet<>(); @@ -575,9 +573,13 @@ private Map importAllComponentsAsPackages(Map importAllComponentsAsPackages(Map messageMap = new HashMap<>(); messageMap.put(DUPLICATE_COMPONENT, String.join(JOINER, duplicateComponents)); + messageMap.put(INVALID_VCS, String.join(JOINER, invalidVCS)); messageMap.put(DUPLICATE_RELEASE, String.join(JOINER, duplicateReleases)); messageMap.put(DUPLICATE_PACKAGE, String.join(JOINER, duplicatePackages)); messageMap.put(INVALID_RELEASE, String.join(JOINER, invalidReleases)); @@ -908,8 +911,10 @@ private String getComponentNameFromVCS(String vcsUrl){ Matcher firstSlashMatcher = FIRST_SLASH_PATTERN.matcher(compName); if (firstSlashMatcher.find()) { compName = firstSlashMatcher.group(1); - compName = StringUtils.substringBefore(compName, HASH); compName = compName.replaceAll(SLASH, "."); + if (vcsUrl.toLowerCase().contains("github.com")) { + compName = compName.replaceAll("\\.git.*", "").replaceAll("#.*", ""); + } } } diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java index 9b1f2af83d..c222bccc94 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java @@ -76,6 +76,7 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.*; @@ -455,27 +456,40 @@ private void setMainLicenses(Component component) { * Add new release to the database */ public AddDocumentRequestSummary addComponent(Component component, String user) throws SW360Exception { - if(isDuplicateUsingVcs(component, true)){ - final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); - Set duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true); - if (duplicates.size() == 1) { - duplicates.forEach(addDocumentRequestSummary::setId); + if (isNotNullEmptyOrWhitespace(component.getVcs())) { + String vcsUrl = component.getVcs(); + + try { + vcsUrl = validateVCS(vcsUrl); + component.setVcs(vcsUrl); + } catch (SW360Exception e) { + return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.INVALID_INPUT); } - return addDocumentRequestSummary; - }else if(isDuplicate(component, true)) { - final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); - Set duplicates = componentRepository.getComponentIdsByName(component.getName(), true); - if (duplicates.size() == 1) { - duplicates.forEach(addDocumentRequestSummary::setId); + if (isDuplicateUsingVcs(vcsUrl, true)){ + final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() + .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); + Set duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true); + if (duplicates.size() == 1) { + duplicates.forEach(addDocumentRequestSummary::setId); + } + return addDocumentRequestSummary; + } - return addDocumentRequestSummary; } - if(component.getName().trim().length() == 0) { - return new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.NAMINGERROR); + + if (component.getName().trim().length() == 0) { + return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.NAMINGERROR); + } else { + if (isDuplicate(component.getName(), true)) { + final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() + .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); + Set duplicates = componentRepository.getComponentIdsByName(component.getName(), true); + if (duplicates.size() == 1) { + duplicates.forEach(addDocumentRequestSummary::setId); + } + return addDocumentRequestSummary; + } } if (!isDependenciesExistInComponent(component)) { @@ -589,31 +603,31 @@ public AddDocumentRequestSummary addRelease(Release release, User user) throws S .setId(id); } - private boolean isDuplicate(Component component, boolean caseInsenstive){ - return isDuplicate(component.getName(), caseInsenstive); + private boolean isDuplicate(Component component, boolean caseInsensitive){ + if (isNotNullEmptyOrWhitespace(component.getVcs())) { + return isDuplicate(component.getName(), caseInsensitive) && isDuplicateUsingVcs(component.getVcs(), caseInsensitive); + } + + return isDuplicate(component.getName(), caseInsensitive); } private boolean isDuplicate(Release release){ return isDuplicate(release.getName(), release.getVersion()); } - private boolean isDuplicate(String componentName, boolean caseInsenstive) { + private boolean isDuplicate(String componentName, boolean caseInsensitive) { if (isNullEmptyOrWhitespace(componentName)) { return false; } - Set duplicates = componentRepository.getComponentIdsByName(componentName, caseInsenstive); + Set duplicates = componentRepository.getComponentIdsByName(componentName, caseInsensitive); return duplicates.size()>0; } - private boolean isDuplicateUsingVcs(Component component, boolean caseInsenstive){ - return isDuplicateUsingVcs(component.getVcs(), caseInsenstive); - } - - private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsenstive){ + private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsensitive){ if (isNullEmptyOrWhitespace(vcsUrl)) { return false; } - Set duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsenstive); + Set duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsensitive); return duplicates.size()>0; } @@ -625,9 +639,9 @@ private boolean isDuplicate(String releaseName, String releaseVersion) { return duplicates.size()>0; } - private void isDuplicateComponent(List componentNames, boolean caseInsenstive) { + private void isDuplicateComponent(List componentNames, boolean caseInsensitive) { for (String name : componentNames) { - if(!isDuplicate(name, caseInsenstive)) + if(!isDuplicate(name, caseInsensitive)) listComponentName.add(name); } } @@ -701,6 +715,17 @@ public RequestStatus updateComponent(Component component, User user, boolean for if (categories == null || categories.isEmpty()) { component.setCategories(ImmutableSet.of(DEFAULT_CATEGORY)); } + + if (isNotNullEmptyOrWhitespace(component.getVcs())) { + try { + String vcsUrl = component.getVcs(); + vcsUrl = validateVCS(vcsUrl); + component.setVcs(vcsUrl); + } catch (SW360Exception e) { + return RequestStatus.INVALID_INPUT; + } + } + // Prepare component for database prepareComponent(component); @@ -823,9 +848,13 @@ private void updateEccStatusForRelease(Component component) { private boolean changeWouldResultInDuplicate(Component before, Component after) { if (before.getName().equals(after.getName())) { - // sth else was changed, not one of the duplication relevant properties return false; } + if (isNotNullEmptyOrWhitespace(before.getVcs())) { + if (before.getVcs().equals(after.getVcs())) { + return false; + } + } return isDuplicate(after, false); } @@ -3190,4 +3219,28 @@ private AttachmentContent makeAttachmentContent(AttachmentContent content) { throw new RuntimeException(e); } } + + public String validateVCS(String vcs) throws SW360Exception { + // GitHub repository URL Format: https://github.com/supplier/name + if (vcs.toLowerCase().contains("github.com")) { + URI uri = URI.create(vcs); + String[] urlParts = uri.getPath().split("/"); + if (urlParts.length >= 3) { + String firstSegment = urlParts[1]; + String secondSegment = urlParts[2].replaceAll("\\.git.*", "").replaceAll("#.*", ""); // Remove .git and anything after, and also remove anything after # + String sanitizedVCS = "https://github.com/" + firstSegment + "/" + secondSegment; + return sanitizedVCS; + } else { + log.error("Invalid GitHub repository URL: " + vcs); + throw new SW360Exception("Invalid GitHub repository URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST); + } + } else { + //repository URL formats for other domains yet to be defined + if (!CommonUtils.isValidUrl(vcs)) { + log.error("Invalid VCS URL: " + vcs); + throw new SW360Exception("Invalid VCS URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST); + } + return vcs; + } + } } diff --git a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java index 382c3283cb..121f70f4ac 100644 --- a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java +++ b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java @@ -25,6 +25,7 @@ public class ErrorMessages { public static final String COMPONENT_NOT_ADDED = "Component could not be added."; public static final String COMPONENT_DUPLICATE = "A component with the same name or VCS already exists."; public static final String COMPONENT_NAMING_ERROR = "Name and Categories of component cannot contain only space characters."; + public static final String INVALID_VCS_OR_LINKED_DOCUMENT = "Invalid VCS or linked document"; public static final String RELEASE_NOT_ADDED = "Release could not be added."; public static final String RELEASE_DUPLICATE = "A release with the same name and version already exists."; public static final String RELEASE_NAME_VERSION_ERROR = "Name and version of release cannot contain only space characters."; @@ -71,7 +72,7 @@ public class ErrorMessages { public static final String ERROR_VENDOR = "Error: Invalid vendor Name or Url."; public static final String ERROR_BULK_DELETING = "Error while bulk deleting"; public static final String ERROR_BULK_DELETING_IN_BACKEND = "Error while bulk deleting in backend."; - + public static final String PACKAGE_NOT_ADDED = "Package could not be added."; public static final String PACKAGE_DUPLICATE = "A package with the same name and version already exists."; public static final String PACKAGE_NAME_VERSION_ERROR = "Name and version of the package cannot contain only space characters."; @@ -93,6 +94,7 @@ public class ErrorMessages { .add(COMPONENT_NOT_ADDED) .add(COMPONENT_DUPLICATE) .add(COMPONENT_NAMING_ERROR) + .add(INVALID_VCS_OR_LINKED_DOCUMENT) .add(RELEASE_NOT_ADDED) .add(RELEASE_DUPLICATE) .add(RELEASE_NAME_VERSION_ERROR) diff --git a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java index d9f10a2c80..32ed8a2314 100644 --- a/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java +++ b/frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java @@ -2318,11 +2318,13 @@ public void updateComponent(ActionRequest request, ActionResponse response) thro RequestStatus requestStatus = client.updateComponent(component, user); setSessionMessage(request, requestStatus, "Component", "update", component.getName()); if (RequestStatus.DUPLICATE.equals(requestStatus) || RequestStatus.DUPLICATE_ATTACHMENT.equals(requestStatus) || - RequestStatus.NAMINGERROR.equals(requestStatus)) { + RequestStatus.NAMINGERROR.equals(requestStatus) || RequestStatus.INVALID_INPUT.equals(requestStatus) ) { if(RequestStatus.DUPLICATE.equals(requestStatus)) setSW360SessionError(request, ErrorMessages.COMPONENT_DUPLICATE); else if (RequestStatus.NAMINGERROR.equals(requestStatus)) setSW360SessionError(request, ErrorMessages.COMPONENT_NAMING_ERROR); + else if (RequestStatus.INVALID_INPUT.equals(requestStatus)) + setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT); else setSW360SessionError(request, ErrorMessages.DUPLICATE_ATTACHMENT); response.setRenderParameter(PAGENAME, PAGENAME_EDIT); @@ -2360,6 +2362,11 @@ else if (RequestStatus.NAMINGERROR.equals(requestStatus)) response.setRenderParameter(PAGENAME, PAGENAME_EDIT); prepareRequestForEditAfterDuplicateOrNamingError(request, component); break; + case INVALID_INPUT: + setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT); + response.setRenderParameter(PAGENAME, PAGENAME_EDIT); + prepareRequestForEditAfterDuplicateOrNamingError(request, component); + break; default: setSW360SessionError(request, ErrorMessages.COMPONENT_NOT_ADDED); response.setRenderParameter(PAGENAME, PAGENAME_VIEW); @@ -2417,6 +2424,8 @@ public void updateRelease(ActionRequest request, ActionResponse response) throws setSW360SessionError(request, ErrorMessages.RELEASE_DUPLICATE); else if (RequestStatus.NAMINGERROR.equals(requestStatus)) setSW360SessionError(request, ErrorMessages.RELEASE_NAME_VERSION_ERROR); + else if (RequestStatus.INVALID_INPUT.equals(requestStatus)) + setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT); else setSW360SessionError(request, ErrorMessages.DUPLICATE_ATTACHMENT); response.setRenderParameter(PAGENAME, PAGENAME_EDIT_RELEASE); diff --git a/frontend/sw360-portlet/src/main/resources/content/Language.properties b/frontend/sw360-portlet/src/main/resources/content/Language.properties index 07025fa61e..5e1d6c63b9 100644 --- a/frontend/sw360-portlet/src/main/resources/content/Language.properties +++ b/frontend/sw360-portlet/src/main/resources/content/Language.properties @@ -858,7 +858,7 @@ link.to.project=Link to Project link.to.projects=Link to Projects list.of.components.without.version.information=List of Components without version information list.of.invalid.packages.without.purl.or.name.or.version=List of invalid Packages without purl or name or version -list.of.packages.without.vcs.information=List of Packages without VCS information +list.of.packages.without.vcs.information=List of Packages with invalid or missing VCS information list.view=List View loading=Loading... loading.data.for.step.1.please.wait=Loading data for step 1, please wait...