diff --git a/lib/schema.ex b/lib/schema.ex index 95464c0..2fe3496 100644 --- a/lib/schema.ex +++ b/lib/schema.ex @@ -418,6 +418,14 @@ defmodule Schema do |> reduce_objects() end + # ----------------------------# + # Enrich Event Data Functions # + # ----------------------------# + + def enrich(data, enum_text, observables) do + Schema.Helper.enrich(data, enum_text, observables) + end + # -------------------------------# # Generate Sample Data Functions # # -------------------------------# diff --git a/lib/schema/helper.ex b/lib/schema/helper.ex new file mode 100644 index 0000000..7948e97 --- /dev/null +++ b/lib/schema/helper.ex @@ -0,0 +1,159 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +defmodule Schema.Helper do + @moduledoc """ + Provides helper functions to enrich the event data. + """ + require Logger + + def enrich(data, enum_text, observables) when is_map(data) do + Logger.debug(fn -> + "enrich event: #{inspect(data)}, enum_text: #{enum_text}, observables: #{observables}" + end) + + enrich_class(data["class_uid"], data, enum_text, observables) + end + + # this is not an event + def enrich(data, _enum_text, _observables), do: %{:error => "Not a JSON object", :data => data} + + # missing class_uid + defp enrich_class(nil, data, _enum_text, _observables), + do: %{:error => "Missing class_uid", :data => data} + + defp enrich_class(class_uid, data, enum_text, _observables) do + Logger.debug("enrich class: #{class_uid}") + + # if observables == "true", do: + + case Schema.find_class(class_uid) do + # invalid event class ID + nil -> + %{:error => "Invalid class_uid: #{class_uid}", :data => data} + + class -> + data = type_uid(class_uid, data) + + if enum_text == "true" do + enrich_type(class, data) + else + data + end + end + end + + defp enrich_type(type, data) do + attributes = type[:attributes] + + Enum.reduce(data, %{}, fn {name, value}, acc -> + key = to_atom(name) + + case attributes[key] do + # Attribute name is not defined in the schema + nil -> + Map.put(acc, name, value) + + attribute -> + {name, text} = enrich_attribute(attribute[:type], name, attribute, value) + + if Map.has_key?(attribute, :enum) do + Logger.debug("enrich enum: #{name} = #{text}") + + case attribute[:sibling] do + nil -> + Map.put_new(acc, name, value) + + sibling -> + Map.put_new(acc, name, value) |> Map.put_new(sibling, text) + end + else + Map.put(acc, name, text) + end + end + end) + end + + defp type_uid(class_uid, data) do + case data["activity_id"] do + nil -> + data + + activity_id -> + uid = + if activity_id >= 0 do + Schema.Types.type_uid(class_uid, activity_id) + else + 0 + end + + Map.put(data, "type_uid", uid) + end + end + + defp to_atom(key) when is_atom(key), do: key + defp to_atom(key), do: String.to_atom(key) + + defp enrich_attribute("integer_t", name, attribute, value) do + enrich_integer(attribute[:enum], name, value) + end + + defp enrich_attribute("object_t", name, attribute, value) when is_map(value) do + {name, enrich_type(Schema.object(attribute[:object_type]), value)} + end + + defp enrich_attribute("object_t", name, attribute, value) when is_list(value) do + data = + if attribute[:is_array] and is_map(List.first(value)) do + obj_type = Schema.object(attribute[:object_type]) + + Enum.map(value, fn data -> + enrich_type(obj_type, data) + end) + else + value + end + + {name, data} + end + + defp enrich_attribute(_, name, _attribute, value) do + {name, value} + end + + # Integer value + defp enrich_integer(nil, name, value) do + {name, value} + end + + # Single enum value + defp enrich_integer(enum, name, value) when is_integer(value) do + key = Integer.to_string(value) |> String.to_atom() + + {name, caption(enum[key], value)} + end + + # Array of enum values + defp enrich_integer(enum, name, values) when is_list(values) do + list = + Enum.map(values, fn n -> + key = Integer.to_string(n) |> String.to_atom() + caption(enum[key], key) + end) + + {name, list} + end + + # Non-integer value + defp enrich_integer(_, name, value), + do: {name, value} + + defp caption(nil, value), do: value + defp caption(map, _value), do: map[:caption] +end diff --git a/lib/schema_web/controllers/schema_controller.ex b/lib/schema_web/controllers/schema_controller.ex index 4814a48..3e4476d 100644 --- a/lib/schema_web/controllers/schema_controller.ex +++ b/lib/schema_web/controllers/schema_controller.ex @@ -21,6 +21,9 @@ defmodule SchemaWeb.SchemaController do @verbose "_mode" @spaces "_spaces" + @enum_text "_enum_text" + @observables "_observables" + # ------------------- # Event Schema API's # ------------------- @@ -579,7 +582,7 @@ defmodule SchemaWeb.SchemaController do def export_base_event(conn, params) do profiles = parse_options(profiles(params)) - base_event = Schema.export_base_event (profiles) + base_event = Schema.export_base_event(profiles) send_json_resp(conn, base_event) end @@ -718,17 +721,91 @@ defmodule SchemaWeb.SchemaController do Schema.object_ex(extensions, extension, id, profiles) end - # --------------------------------- - # Validation and translation API's - # --------------------------------- + # --------------------------------------------- + # Enrichment, validation, and translation API's + # --------------------------------------------- + + @doc """ + Enrich event data by adding type_uid, enumerated text, and observables. + A single event is encoded as a JSON object and multiple events are encoded as JSON array of objects. + """ + swagger_path :enrich do + post("/api/enrich") + summary("Enrich Event") + + description(""" + The purpose of this API is to enrich the provided event data with type_uid, enumerated text, and observables array. + Each event is represented as a JSON object, while multiple events are encoded as a JSON array of objects. + """) + + produces("application/json") + tag("Tools") + + parameters do + _enum_text( + :query, + :boolean, + """ + Enhance the event data by adding the enumerated text values.
+ + |Value|Example| + |-----|-------| + |true|Untranslated:
{"category_uid":0,"class_uid":0,"activity_id": 0,"severity_id": 5,"status": "Something else","status_id": 99,"time": 1689125893360905}

Translated:
{"activity_name": "Unknown", "activity_id": 0, "category_name": "Uncategorized", "category_uid": 0, "class_name": "Base Event", "class_uid": 0, "severity": "Critical", "severity_id": 5, "status": "Something else", "status_id": 99, "time": 1689125893360905, "type_name": "Base Event: Unknown", "type_uid": 0}| + """, + default: false + ) + + _observables( + :query, + :boolean, + """ + TODO: Enhance the event data by adding the observables associated with the event. + """, + default: false + ) + + data(:body, :object, "The event data to be enriched.", required: true) + end + + response(200, "Success") + end + + @spec enrich(Plug.Conn.t(), map) :: Plug.Conn.t() + def enrich(conn, data) do + {enum_text, data} = Map.pop(data, @enum_text) + {observables, data} = Map.pop(data, @observables) + + result = + case data["_json"] do + # Enrich a single events + nil -> + Schema.enrich(data, enum_text, observables) + + # Enrich a list of events + list when is_list(list) -> + Enum.map(list, &Task.async(fn -> Schema.enrich(&1, enum_text, observables) end)) + |> Enum.map(&Task.await/1) + + # something other than json data + other -> + %{:error => "The data does not look like an event", :data => other} + end + + send_json_resp(conn, result) + end @doc """ - Translate event data. A single event is encoded as a JSON object and multiple events are encoded as JSON array of object. + Translate event data. A single event is encoded as a JSON object and multiple events are encoded as JSON array of objects. """ swagger_path :translate do post("/api/translate") summary("Translate Event") - description("Translate event data.") + + description(""" + The purpose of this API is to translate the provided event data using the OCSF schema. + Each event is represented as a JSON object, while multiple events are encoded as a JSON array of objects. + """) + produces("application/json") tag("Tools") @@ -775,25 +852,24 @@ defmodule SchemaWeb.SchemaController do def translate(conn, data) do options = [spaces: data[@spaces], verbose: verbose(data[@verbose])] - case data["_json"] do - nil -> + result = + case data["_json"] do # Translate a single events - data = + nil -> Map.delete(data, @verbose) |> Map.delete(@spaces) |> Schema.Translator.translate(options) - send_json_resp(conn, data) - - list when is_list(list) -> # Translate a list of events - translated = Enum.map(list, fn data -> Schema.Translator.translate(data, options) end) - send_json_resp(conn, translated) + list when is_list(list) -> + Enum.map(list, fn data -> Schema.Translator.translate(data, options) end) - other -> # some other json data - send_json_resp(conn, other) - end + other -> + %{:error => "The data does not look like an event", "data" => other} + end + + send_json_resp(conn, result) end @doc """ @@ -804,7 +880,12 @@ defmodule SchemaWeb.SchemaController do swagger_path :validate do post("/api/validate") summary("Validate Event") - description("Validate event data.") + + description(""" + The primary objective of this API is to validate the provided event data against the OCSF schema. + Each event is represented as a JSON object, while multiple events are encoded as a JSON array of objects. + """) + produces("application/json") tag("Tools") @@ -817,23 +898,23 @@ defmodule SchemaWeb.SchemaController do @spec validate(Plug.Conn.t(), map) :: Plug.Conn.t() def validate(conn, data) do - case data["_json"] do - nil -> + result = + case data["_json"] do # Validate a single events - send_json_resp(conn, Schema.Inspector.validate(data)) + nil -> + Schema.Inspector.validate(data) - list when is_list(list) -> # Validate a list of events - result = - list - |> Enum.map(&Task.async(fn -> Schema.Inspector.validate(&1) end)) + list when is_list(list) -> + Enum.map(list, &Task.async(fn -> Schema.Inspector.validate(&1) end)) |> Enum.map(&Task.await/1) - send_json_resp(conn, result) - other -> # some other json data - send_json_resp(conn, %{:error => "The data does not look like an event", "data" => other}) - end + other -> + %{:error => "The data does not look like an event", "data" => other} + end + + send_json_resp(conn, result) end # -------------------------- @@ -1072,5 +1153,4 @@ defmodule SchemaWeb.SchemaController do defp parse_java_package(nil), do: [] defp parse_java_package(""), do: [] defp parse_java_package(name), do: [package_name: name] - end diff --git a/lib/schema_web/router.ex b/lib/schema_web/router.ex index 7a807fb..e71b549 100644 --- a/lib/schema_web/router.ex +++ b/lib/schema_web/router.ex @@ -81,6 +81,7 @@ defmodule SchemaWeb.Router do get "/data_types", SchemaController, :data_types + post "/enrich", SchemaController, :enrich post "/translate", SchemaController, :translate post "/validate", SchemaController, :validate end diff --git a/mix.exs b/mix.exs index ebeaef3..949b8fe 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule Schema.MixProject do use Mix.Project - @version "2.50.1" + @version "2.51.0" def project do build = System.get_env("GITHUB_RUN_NUMBER") || "SNAPSHOT"