Skip to content

Commit

Permalink
feat(ComponentPortletandImportCDX): Validate VCS URL
Browse files Browse the repository at this point in the history 
Signed-off-by: afsahsyeda <afsah.syeda@siemens-healthineers.com>
  • Loading branch information
@afsahsyeda
afsahsyeda committed Jun 11, 2024
1 parent 6d12f46 commit c1480f2
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 54 deletions.
47 changes: 26 additions & 21 deletions backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -25,9 +26,9 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TException;
Expand Down Expand Up @@ -113,6 +114,7 @@ public class CycloneDxBOMImporter {
private static final String INVALID_COMPONENT = "invalidComp";
private static final String INVALID_RELEASE = "invalidRel";
private static final String INVALID_PACKAGE = "invalidPkg";
private static final String INVALID_VCS = "invalidVCS";
private static final String PROJECT_ID = "projectId";
private static final String PROJECT_NAME = "projectName";
private static final boolean IS_PACKAGE_PORTLET_ENABLED = SW360Constants.IS_PACKAGE_PORTLET_ENABLED;
Expand Down Expand Up @@ -142,13 +144,11 @@ public CycloneDxBOMImporter(ProjectDatabaseHandler projectDatabaseHandler, Compo
* @return Map<String, List<org.cyclonedx.model.Component>>
*/
private Map<String, List<org.cyclonedx.model.Component>> getVcsToComponentMap(List<org.cyclonedx.model.Component> components) {
return components.stream().filter(Objects::nonNull)
return components.parallelStream().filter(Objects::nonNull)
.flatMap(comp -> CommonUtils.nullToEmptyList(comp.getExternalReferences()).stream()
.filter(Objects::nonNull)
.filter(ref -> ExternalReference.Type.VCS.equals(ref.getType()))
.map(ExternalReference::getUrl)
.map(String::toLowerCase)
.map(url -> StringUtils.removeEnd(url, DOT_GIT))
.map(url -> new AbstractMap.SimpleEntry<>(url, comp)))
.collect(Collectors.groupingBy(e -> e.getKey(),
Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
Expand Down Expand Up @@ -190,21 +190,15 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
.filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count();
long componentsCount = components.size();
org.cyclonedx.model.Component compMetadata = bomMetadata.getComponent();
Map<String, List<org.cyclonedx.model.Component>> vcsToComponentMap = new HashMap<>();
final Map<String, List<org.cyclonedx.model.Component>> vcsToComponentMap = new HashMap<>();

if (!IS_PACKAGE_PORTLET_ENABLED) {
vcsToComponentMap.put("", components);
requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
} else {
vcsToComponentMap.putAll(getVcsToComponentMap(components));

vcsToComponentMap = getVcsToComponentMap(components);
if (componentsCount == vcsCount) {

requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
} else if (componentsCount > vcsCount) {

requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);

requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
if (requestSummary.requestStatus.equals(RequestStatus.SUCCESS)) {

String jsonMessage = requestSummary.getMessage();
Expand All @@ -214,6 +208,14 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
if (CommonUtils.isNullEmptyOrWhitespace(projId)) {
return requestSummary;
}
final List<org.cyclonedx.model.Component> componentsWithInvalidVCS = new ArrayList<>();

if (CommonUtils.isNotNullEmptyOrWhitespace(messageMap.get(INVALID_VCS))) {
componentsWithInvalidVCS.addAll(Stream.of(messageMap.get(INVALID_VCS).split("\\|\\|"))
.map(vcs -> vcsToComponentMap.get(vcs)).flatMap(List::stream)
.collect(Collectors.toList()));
}

final Set<String> duplicatePackages = new HashSet<>();
final Set<String> componentsWithoutVcs = new HashSet<>();
final Set<String> invalidPackages = new HashSet<>();
Expand All @@ -236,7 +238,8 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a

for (org.cyclonedx.model.Component comp : components) {
if (CommonUtils.isNullOrEmptyCollection(comp.getExternalReferences())
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0) {
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0
|| componentsWithInvalidVCS.contains(comp)) {

final var fullName = SW360Utils.getVersionedName(comp.getName(), comp.getVersion());
final var licenses = getLicenseFromBomComponent(comp);
Expand Down Expand Up @@ -297,13 +300,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
messageMap.put(PKG_REUSE_COUNT_KEY, String.valueOf(pkgReuseCount));
requestSummary.setMessage(convertCollectionToJSONString(messageMap));
}
} else {
requestSummary.setMessage(String.format(String.format(
"SBOM import aborted with error: Multiple vcs information found in compnents, vcs found: %s and total components: %s",
vcsCount, componentsCount)));
return requestSummary;
}
}

if (RequestStatus.SUCCESS.equals(requestSummary.getRequestStatus())) {
String jsonMessage = requestSummary.getMessage();
Expand Down Expand Up @@ -553,6 +550,7 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c

final var countMap = new HashMap<String, Integer>();
final Set<String> duplicateComponents = new HashSet<>();
final Set<String> invalidVCS = new HashSet<>();
final Set<String> duplicateReleases = new HashSet<>();
final Set<String> duplicatePackages = new HashSet<>();
final Set<String> invalidReleases = new HashSet<>();
Expand All @@ -575,9 +573,13 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c
String existingCompName = getComponetNameById(comp.getId(), user);
comp.setName(existingCompName);
} else {
if (compAddSummary.getRequestStatus().equals(AddDocumentRequestStatus.DUPLICATE)) {
// in case of more than 1 duplicate found, then continue and show error message in UI.
log.warn("found multiple components: " + comp.getName());
duplicateComponents.add(comp.getName());
} else if (compAddSummary.getRequestStatus().equals(AddDocumentRequestStatus.INVALID_INPUT)) {
invalidVCS.add(entry.getKey());
}
continue;
}

Expand Down Expand Up @@ -677,6 +679,7 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c
project.setReleaseIdToUsage(releaseRelationMap);
final Map<String, String> messageMap = new HashMap<>();
messageMap.put(DUPLICATE_COMPONENT, String.join(JOINER, duplicateComponents));
messageMap.put(INVALID_VCS, String.join(JOINER, invalidVCS));
messageMap.put(DUPLICATE_RELEASE, String.join(JOINER, duplicateReleases));
messageMap.put(DUPLICATE_PACKAGE, String.join(JOINER, duplicatePackages));
messageMap.put(INVALID_RELEASE, String.join(JOINER, invalidReleases));
Expand Down Expand Up @@ -908,8 +911,10 @@ private String getComponentNameFromVCS(String vcsUrl){
Matcher firstSlashMatcher = FIRST_SLASH_PATTERN.matcher(compName);
if (firstSlashMatcher.find()) {
compName = firstSlashMatcher.group(1);
compName = StringUtils.substringBefore(compName, HASH);
compName = compName.replaceAll(SLASH, ".");
if (vcsUrl.toLowerCase().contains("github.com")) {
compName = compName.replaceAll("\\.git.*", "").replaceAll("#.*", "");
}
}
}

Expand Down
113 changes: 83 additions & 30 deletions ...d/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@

import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.file.*;
Expand Down Expand Up @@ -455,27 +456,40 @@ private void setMainLicenses(Component component) {
* Add new release to the database
*/
public AddDocumentRequestSummary addComponent(Component component, String user) throws SW360Exception {
if(isDuplicateUsingVcs(component, true)){
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true);
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
if (isNotNullEmptyOrWhitespace(component.getVcs())) {
String vcsUrl = component.getVcs();

try {
vcsUrl = validateVCS(vcsUrl);
component.setVcs(vcsUrl);
} catch (SW360Exception e) {
return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.INVALID_INPUT);
}
return addDocumentRequestSummary;

}else if(isDuplicate(component, true)) {
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByName(component.getName(), true);
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
if (isDuplicateUsingVcs(vcsUrl, true)){
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true);
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
}
return addDocumentRequestSummary;

}
return addDocumentRequestSummary;
}
if(component.getName().trim().length() == 0) {
return new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.NAMINGERROR);

if (component.getName().trim().length() == 0) {
return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.NAMINGERROR);
} else {
if (isDuplicate(component.getName(), true)) {
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByName(component.getName(), true);
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
}
return addDocumentRequestSummary;
}
}

if (!isDependenciesExistInComponent(component)) {
Expand Down Expand Up @@ -589,31 +603,31 @@ public AddDocumentRequestSummary addRelease(Release release, User user) throws S
.setId(id);
}

private boolean isDuplicate(Component component, boolean caseInsenstive){
return isDuplicate(component.getName(), caseInsenstive);
private boolean isDuplicate(Component component, boolean caseInsensitive){
if (isNotNullEmptyOrWhitespace(component.getVcs())) {
return isDuplicate(component.getName(), caseInsensitive) && isDuplicateUsingVcs(component.getVcs(), caseInsensitive);
}

return isDuplicate(component.getName(), caseInsensitive);
}

private boolean isDuplicate(Release release){
return isDuplicate(release.getName(), release.getVersion());
}

private boolean isDuplicate(String componentName, boolean caseInsenstive) {
private boolean isDuplicate(String componentName, boolean caseInsensitive) {
if (isNullEmptyOrWhitespace(componentName)) {
return false;
}
Set<String> duplicates = componentRepository.getComponentIdsByName(componentName, caseInsenstive);
Set<String> duplicates = componentRepository.getComponentIdsByName(componentName, caseInsensitive);
return duplicates.size()>0;
}

private boolean isDuplicateUsingVcs(Component component, boolean caseInsenstive){
return isDuplicateUsingVcs(component.getVcs(), caseInsenstive);
}

private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsenstive){
private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsensitive){
if (isNullEmptyOrWhitespace(vcsUrl)) {
return false;
}
Set<String> duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsenstive);
Set<String> duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsensitive);
return duplicates.size()>0;
}

Expand All @@ -625,9 +639,9 @@ private boolean isDuplicate(String releaseName, String releaseVersion) {
return duplicates.size()>0;
}

private void isDuplicateComponent(List<String> componentNames, boolean caseInsenstive) {
private void isDuplicateComponent(List<String> componentNames, boolean caseInsensitive) {
for (String name : componentNames) {
if(!isDuplicate(name, caseInsenstive))
if(!isDuplicate(name, caseInsensitive))
listComponentName.add(name);
}
}
Expand Down Expand Up @@ -701,6 +715,17 @@ public RequestStatus updateComponent(Component component, User user, boolean for
if (categories == null || categories.isEmpty()) {
component.setCategories(ImmutableSet.of(DEFAULT_CATEGORY));
}

if (isNotNullEmptyOrWhitespace(component.getVcs())) {
try {
String vcsUrl = component.getVcs();
vcsUrl = validateVCS(vcsUrl);
component.setVcs(vcsUrl);
} catch (SW360Exception e) {
return RequestStatus.INVALID_INPUT;
}
}

// Prepare component for database
prepareComponent(component);

Expand Down Expand Up @@ -823,9 +848,13 @@ private void updateEccStatusForRelease(Component component) {

private boolean changeWouldResultInDuplicate(Component before, Component after) {
if (before.getName().equals(after.getName())) {
// sth else was changed, not one of the duplication relevant properties
return false;
}
if (isNotNullEmptyOrWhitespace(before.getVcs())) {
if (before.getVcs().equals(after.getVcs())) {
return false;
}
}

return isDuplicate(after, false);
}
Expand Down Expand Up @@ -3190,4 +3219,28 @@ private AttachmentContent makeAttachmentContent(AttachmentContent content) {
throw new RuntimeException(e);
}
}

public String validateVCS(String vcs) throws SW360Exception {
// GitHub repository URL Format: https://github.com/supplier/name
if (vcs.toLowerCase().contains("github.com")) {
URI uri = URI.create(vcs);
String[] urlParts = uri.getPath().split("/");
if (urlParts.length >= 3) {
String firstSegment = urlParts[1];
String secondSegment = urlParts[2].replaceAll("\\.git.*", "").replaceAll("#.*", ""); // Remove .git and anything after, and also remove anything after #
String sanitizedVCS = "https://github.com/" + firstSegment + "/" + secondSegment;
return sanitizedVCS;
} else {
log.error("Invalid GitHub repository URL: " + vcs);
throw new SW360Exception("Invalid GitHub repository URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST);
}
} else {
//repository URL formats for other domains yet to be defined
if (!CommonUtils.isValidUrl(vcs)) {
log.error("Invalid VCS URL: " + vcs);
throw new SW360Exception("Invalid VCS URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST);
}
return vcs;
}
}
}
4 changes: 3 additions & 1 deletion frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class ErrorMessages {
public static final String COMPONENT_NOT_ADDED = "Component could not be added.";
public static final String COMPONENT_DUPLICATE = "A component with the same name or VCS already exists.";
public static final String COMPONENT_NAMING_ERROR = "Name and Categories of component cannot contain only space characters.";
public static final String INVALID_VCS_OR_LINKED_DOCUMENT = "Invalid VCS or linked document";
public static final String RELEASE_NOT_ADDED = "Release could not be added.";
public static final String RELEASE_DUPLICATE = "A release with the same name and version already exists.";
public static final String RELEASE_NAME_VERSION_ERROR = "Name and version of release cannot contain only space characters.";
Expand Down Expand Up @@ -71,7 +72,7 @@ public class ErrorMessages {
public static final String ERROR_VENDOR = "Error: Invalid vendor Name or Url.";
public static final String ERROR_BULK_DELETING = "Error while bulk deleting";
public static final String ERROR_BULK_DELETING_IN_BACKEND = "Error while bulk deleting in backend.";

public static final String PACKAGE_NOT_ADDED = "Package could not be added.";
public static final String PACKAGE_DUPLICATE = "A package with the same name and version already exists.";
public static final String PACKAGE_NAME_VERSION_ERROR = "Name and version of the package cannot contain only space characters.";
Expand All @@ -93,6 +94,7 @@ public class ErrorMessages {
.add(COMPONENT_NOT_ADDED)
.add(COMPONENT_DUPLICATE)
.add(COMPONENT_NAMING_ERROR)
.add(INVALID_VCS_OR_LINKED_DOCUMENT)
.add(RELEASE_NOT_ADDED)
.add(RELEASE_DUPLICATE)
.add(RELEASE_NAME_VERSION_ERROR)
Expand Down
Loading

0 comments on commit c1480f2

Please sign in to comment.