Protobuf schema references support

.gitignore

@@ -16,3 +16,4 @@ __pycache__/
 /kafka_*/
 venv
 /karapace/version.py
+.run

karapace/dependency.py

@@ -0,0 +1,47 @@
+from karapace.schema_references import Reference
+from karapace.typing import JsonData, Subject, Version
+from typing import Any, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from karapace.schema_models import ValidatedTypedSchema
+
+
+class DependencyVerifierResult:
+    def __init__(self, result: bool, message: Optional[str] = "") -> None:
+        self.result = result
+        self.message = message
+
+
+class Dependency:
+    def __init__(self, name: str, subject: Subject, version: Version, target_schema: "ValidatedTypedSchema") -> None:
+        self.name = name
+        self.subject = subject
+        self.version = version
+        self.schema = target_schema
+
+    @staticmethod
+    def of(reference: Reference, target_schema: "ValidatedTypedSchema") -> "Dependency":
+        return Dependency(reference.name, reference.subject, reference.version, target_schema)
+
+    def to_dict(self) -> JsonData:
+        return {
+            "name": self.name,
+            "subject": self.subject,
+            "version": self.version,
+        }
+
+    def identifier(self) -> str:
+        return self.name + "_" + self.subject + "_" + str(self.version)
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.subject, self.version, self.schema))
+
+    def __eq__(self, other: Any) -> bool:
+        if other is None or not isinstance(other, Dependency):
+            return False
+        return (
+            self.name == other.name
+            and self.subject == other.subject
+            and self.version == other.version
+            and self.schema == other.schema
+        )

karapace/errors.py

@@ -1,3 +1,7 @@
+from karapace.typing import Version
+from typing import List
+
+
 class VersionNotFoundException(Exception):
     pass
 
@@ -18,6 +22,10 @@ class InvalidSchemaType(Exception):
     pass
 
 
+class InvalidReferences(Exception):
+    pass
+
+
 class SchemasNotFoundException(Exception):
     pass
 
@@ -38,6 +46,13 @@ class SubjectNotSoftDeletedException(Exception):
     pass
 
 
+class ReferenceExistsException(Exception):
+    def __init__(self, referenced_by: List, version: Version):
+        super().__init__()
+        self.version = version
+        self.referenced_by = referenced_by
+
+
 class SubjectSoftDeletedException(Exception):
     pass
 

karapace/protobuf/dependency.py

@@ -0,0 +1,56 @@
+from karapace.dependency import DependencyVerifierResult
+from karapace.protobuf.known_dependency import DependenciesHardcoded, KnownDependency
+from karapace.protobuf.one_of_element import OneOfElement
+from typing import List
+
+
+class ProtobufDependencyVerifier:
+    def __init__(self) -> None:
+        self.declared_types: List[str] = []
+        self.used_types: List[str] = []
+        self.import_path: List[str] = []
+
+    def add_declared_type(self, full_name: str) -> None:
+        self.declared_types.append(full_name)
+
+    def add_used_type(self, parent: str, element_type: str) -> None:
+        if element_type.find("map<") == 0:
+            end = element_type.find(">")
+            virgule = element_type.find(",")
+            key = element_type[4:virgule]
+            value = element_type[virgule + 1 : end]
+            value = value.strip()
+            self.used_types.append(parent + ";" + key)
+            self.used_types.append(parent + ";" + value)
+        else:
+            self.used_types.append(parent + ";" + element_type)
+
+    def add_import(self, import_name: str) -> None:
+        self.import_path.append(import_name)
+
+    def verify(self) -> DependencyVerifierResult:
+        declared_index = set(self.declared_types)
+        for used_type in self.used_types:
+            delimiter = used_type.rfind(";")
+            used_type_with_scope = ""
+            if delimiter != -1:
+                used_type_with_scope = used_type[:delimiter] + "." + used_type[delimiter + 1 :]
+                used_type = used_type[delimiter + 1 :]
+
+            if not (
+                used_type in DependenciesHardcoded.index
+                or KnownDependency.index_simple.get(used_type) is not None
+                or KnownDependency.index.get(used_type) is not None
+                or used_type in declared_index
+                or (delimiter != -1 and used_type_with_scope in declared_index)
+                or "." + used_type in declared_index
+            ):
+                return DependencyVerifierResult(False, f"type {used_type} is not defined")
+
+        return DependencyVerifierResult(True)
+
+
+def _process_one_of(verifier: ProtobufDependencyVerifier, package_name: str, parent_name: str, one_of: OneOfElement) -> None:
+    parent = package_name + "." + parent_name
+    for field in one_of.fields:
+        verifier.add_used_type(parent, field.element_type)

karapace/protobuf/exception.py

@@ -25,6 +25,10 @@ class ProtobufTypeException(Error):
     """Generic Protobuf type error."""
 
 
+class ProtobufUnresolvedDependencyException(ProtobufException):
+    """a Protobuf schema has unresolved dependency"""
+
+
 class SchemaParseException(ProtobufException):
     """Error while parsing a Protobuf schema descriptor."""
 

karapace/protobuf/field_element.py

@@ -143,10 +143,17 @@ def compare_message(
 
         self_type_record = types.get_self_type(self_type)
         other_type_record = types.get_other_type(other_type)
+
         self_type_element: MessageElement = self_type_record.type_element
         other_type_element: MessageElement = other_type_record.type_element
 
         if types.self_type_short_name(self_type) != types.other_type_short_name(other_type):
             result.add_modification(Modification.FIELD_NAME_ALTER)
         else:
             self_type_element.compare(other_type_element, result, types)
+
+    def __repr__(self):
+        return f"{self.element_type} {self.name} = {self.tag}"
+
+    def __str__(self):
+        return f"{self.element_type} {self.name} = {self.tag}"

karapace/protobuf/known_dependency.py

@@ -0,0 +1,154 @@
+# Support of known dependencies
+
+from enum import Enum
+from typing import Any, Dict, Set
+
+
+def static_init(cls: Any) -> object:
+    if getattr(cls, "static_init", None):
+        cls.static_init()
+    return cls
+
+
+class KnownDependencyLocation(Enum):
+    ANY_LOCATION = "google/protobuf/any.proto"
+    API_LOCATION = "google/protobuf/api.proto"
+    DESCRIPTOR_LOCATION = "google/protobuf/descriptor.proto"
+    DURATION_LOCATION = "google/protobuf/duration.proto"
+    EMPTY_LOCATION = "google/protobuf/empty.proto"
+    FIELD_MASK_LOCATION = "google/protobuf/field_mask.proto"
+    SOURCE_CONTEXT_LOCATION = "google/protobuf/source_context.proto"
+    STRUCT_LOCATION = "google/protobuf/struct.proto"
+    TIMESTAMP_LOCATION = "google/protobuf/timestamp.proto"
+    TYPE_LOCATION = "google/protobuf/type.proto"
+    WRAPPER_LOCATION = "google/protobuf/wrappers.proto"
+    CALENDAR_PERIOD_LOCATION = "google/type/calendar_period.proto"
+    COLOR_LOCATION = "google/type/color.proto"
+    DATE_LOCATION = "google/type/date.proto"
+    DATETIME_LOCATION = "google/type/datetime.proto"
+    DAY_OF_WEEK_LOCATION = "google/type/dayofweek.proto"
+    DECIMAL_LOCATION = "google/type/decimal.proto"
+    EXPR_LOCATION = "google/type/expr.proto"
+    FRACTION_LOCATION = "google/type/fraction.proto"
+    INTERVAL_LOCATION = "google/type/interval.proto"
+    LATLNG_LOCATION = "google/type/latlng.proto"
+    MONEY_LOCATION = "google/type/money.proto"
+    MONTH_LOCATION = "google/type/month.proto"
+    PHONE_NUMBER_LOCATION = "google/type/phone_number.proto"
+    POSTAL_ADDRESS_LOCATION = "google/type/postal_address.proto"
+    QUATERNION_LOCATION = "google/type/quaternion.proto"
+    TIME_OF_DAY_LOCATION = "google/type/timeofday.proto"
+
+
+@static_init
+class KnownDependency:
+    index: Dict = dict()
+    index_simple: Dict = dict()
+    map: Dict = {
+        "google/protobuf/any.proto": ["google.protobuf.Any"],
+        "google/protobuf/api.proto": ["google.protobuf.Api", "google.protobuf.Method", "google.protobuf.Mixin"],
+        "google/protobuf/descriptor.proto": [
+            "google.protobuf.FileDescriptorSet",
+            "google.protobuf.FileDescriptorProto",
+            "google.protobuf.DescriptorProto",
+            "google.protobuf.ExtensionRangeOptions",
+            "google.protobuf.FieldDescriptorProto",
+            "google.protobuf.OneofDescriptorProto",
+            "google.protobuf.EnumDescriptorProto",
+            "google.protobuf.EnumValueDescriptorProto",
+            "google.protobuf.ServiceDescriptorProto",
+            "google.protobuf.MethodDescriptorProto",
+            "google.protobuf.FileOptions",
+            "google.protobuf.MessageOptions",
+            "google.protobuf.FieldOptions",
+            "google.protobuf.OneofOptions",
+            "google.protobuf.EnumOptions",
+            "google.protobuf.EnumValueOptions",
+            "google.protobuf.ServiceOptions",
+            "google.protobuf.MethodOptions",
+            "google.protobuf.UninterpretedOption",
+            "google.protobuf.SourceCodeInfo",
+            "google.protobuf.GeneratedCodeInfo",
+        ],
+        "google/protobuf/duration.proto": ["google.protobuf.Duration"],
+        "google/protobuf/empty.proto": ["google.protobuf.Empty"],
+        "google/protobuf/field_mask.proto": ["google.protobuf.FieldMask"],
+        "google/protobuf/source_context.proto": ["google.protobuf.SourceContext"],
+        "google/protobuf/struct.proto": [
+            "google.protobuf.Struct",
+            "google.protobuf.Value",
+            "google.protobuf.NullValue",
+            "google.protobuf.ListValue",
+        ],
+        "google/protobuf/timestamp.proto": ["google.protobuf.Timestamp"],
+        "google/protobuf/type.proto": [
+            "google.protobuf.Type",
+            "google.protobuf.Field",
+            "google.protobuf.Enum",
+            "google.protobuf.EnumValue",
+            "google.protobuf.Option",
+            "google.protobuf.Syntax",
+        ],
+        "google/protobuf/wrappers.proto": [
+            "google.protobuf.DoubleValue",
+            "google.protobuf.FloatValue",
+            "google.protobuf.Int64Value",
+            "google.protobuf.UInt64Value",
+            "google.protobuf.Int32Value",
+            "google.protobuf.UInt32Value",
+            "google.protobuf.BoolValue",
+            "google.protobuf.StringValue",
+            "google.protobuf.BytesValue",
+        ],
+        "google/type/calendar_period.proto": ["google.type.CalendarPeriod"],
+        "google/type/color.proto": ["google.type.Color"],
+        "google/type/date.proto": ["google.type.Date"],
+        "google/type/datetime.proto": ["google.type.DateTime", "google.type.TimeZone"],
+        "google/type/dayofweek.proto": ["google.type.DayOfWeek"],
+        "google/type/decimal.proto": ["google.type.Decimal"],
+        "google/type/expr.proto": ["google.type.Expr"],
+        "google/type/fraction.proto": ["google.type.Fraction"],
+        "google/type/interval.proto": ["google.type.Interval"],
+        "google/type/latlng.proto": ["google.type.LatLng"],
+        "google/type/money.proto": ["google.type.Money"],
+        "google/type/month.proto": ["google.type.Month"],
+        "google/type/phone_number.proto": ["google.type.PhoneNumber"],
+        "google/type/postal_address.proto": ["google.type.PostalAddress"],
+        "google/type/quaternion.proto": ["google.type.Quaternion"],
+        "google/type/timeofday.proto": ["google.type.TimeOfDay"],
+        "confluent/meta.proto": [".confluent.Meta"],
+        "confluent/type/decimal.proto": [".confluent.type.Decimal"],
+    }
+
+    @classmethod
+    def static_init(cls) -> None:
+        for key, value in cls.map.items():
+            for item in value:
+                cls.index[item] = key
+                dot = item.rfind(".")
+                cls.index_simple[item[dot + 1 :]] = key
+
+
+@static_init
+class DependenciesHardcoded:
+    index: Set = set()
+
+    @classmethod
+    def static_init(cls) -> None:
+        cls.index = {
+            "bool",
+            "bytes",
+            "double",
+            "float",
+            "fixed32",
+            "fixed64",
+            "int32",
+            "int64",
+            "sfixed32",
+            "sfixed64",
+            "sint32",
+            "sint64",
+            "string",
+            "uint32",
+            "uint64",
+        }

karapace/protobuf/proto_file_element.py

@@ -9,23 +9,24 @@
 from karapace.protobuf.message_element import MessageElement
 from karapace.protobuf.syntax import Syntax
 from karapace.protobuf.type_element import TypeElement
+from typing import List, Optional
 
 
 class ProtoFileElement:
     def __init__(
         self,
         location: Location,
-        package_name: str = None,
-        syntax: Syntax = None,
-        imports: list = None,
-        public_imports: list = None,
-        types=None,
-        services: list = None,
-        extend_declarations: list = None,
-        options: list = None,
+        package_name: Optional[str] = None,
+        syntax: Optional[Syntax] = None,
+        imports: Optional[list] = None,
+        public_imports: Optional[list] = None,
+        types: Optional[List[TypeElement]] = None,
+        services: Optional[list] = None,
+        extend_declarations: Optional[list] = None,
+        options: Optional[list] = None,
     ) -> None:
         if types is None:
-            types = []
+            types = list()
         self.location = location
         self.package_name = package_name
         self.syntax = syntax
@@ -98,8 +99,13 @@ def __eq__(self, other: "ProtoFileElement") -> bool:  # type: ignore
     def __repr__(self) -> str:
         return self.to_schema()
 
-    def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareResult:
-
+    def compare(
+        self,
+        other: "ProtoFileElement",
+        result: CompareResult,
+        self_dependency_types: Optional[List[TypeElement]] = None,
+        other_dependency_types: Optional[List[TypeElement]] = None,
+    ) -> CompareResult:
         if self.package_name != other.package_name:
             result.add_modification(Modification.PACKAGE_ALTER)
         # TODO: do we need syntax check?
@@ -124,6 +130,22 @@ def compare(self, other: "ProtoFileElement", result: CompareResult) -> CompareRe
             package_name = other.package_name or ""
             compare_types.add_other_type(package_name, type_)
 
+        # If there are dependencies declared, add the types for both.
+        if self_dependency_types:
+            for i, type_ in enumerate(self_dependency_types):
+                package_name = ""
+
+                self_types[type_.name] = type_
+                self_indexes[type_.name] = i
+                compare_types.add_self_type(package_name, type_)
+
+        if other_dependency_types:
+            for i, type_ in enumerate(other_dependency_types):
+                package_name = ""
+                other_types[type_.name] = type_
+                other_indexes[type_.name] = i
+                compare_types.add_other_type(package_name, type_)
+
         for name in chain(self_types.keys(), other_types.keys() - self_types.keys()):
 
             result.push_path(str(name), True)