feat(pm4py): continuing major refactoring

examples/activity_position.py

@@ -1,12 +1,12 @@
-import pandas as pd
 import pm4py
 from pm4py.util import constants, pandas_utils
+from pm4py.objects.log.util import dataframe_utils
 import os
 
 
 def execute_script():
     dataframe = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "receipt.csv"))
-    dataframe["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(dataframe["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    dataframe = dataframe_utils.convert_timestamp_columns_in_df(dataframe, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     # prints the summary of the positions of two activities
     print(pm4py.get_activity_position_summary(dataframe, "Confirmation of receipt"))
     print(pm4py.get_activity_position_summary(dataframe, "T02 Check confirmation of receipt"))

examples/corr_mining.py

@@ -1,4 +1,3 @@
-import pandas as pd
 from pm4py.util import constants, pandas_utils
 
 from pm4py.algo.discovery.correlation_mining import algorithm as correlation_miner
@@ -13,7 +12,7 @@
 
 def execute_script():
     df = pandas_utils.read_csv("../tests/input_data/interval_event_log.csv")
-    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
     act_count = dict(df["concept:name"].value_counts())
     parameters = {}
     parameters[constants.PARAMETER_CONSTANT_START_TIMESTAMP_KEY] = "start_timestamp"

examples/dataframe_prefix_and_fea_extraction.py

@@ -1,13 +1,11 @@
-import pandas as pd
-
 from pm4py.objects.log.util import dataframe_utils
 from pm4py.util import pandas_utils, constants
 
 
 def execute_script():
     # loads a dataframe. setup dates
     df = pandas_utils.read_csv("../tests/input_data/receipt.csv")
-    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
     print(df)
     # insert the case index in the dataframe
     df = pandas_utils.insert_ev_in_tr_index(df, case_id="case:concept:name", column_name="@@index_in_trace")

examples/df_to_log_postpro.py

@@ -1,13 +1,12 @@
 import pm4py
-import pandas as pd
 from pm4py.util import constants, pandas_utils
 from pm4py.objects.conversion.log import converter as log_converter
 import os
 
 
 def execute_script():
     dataframe = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "running-example.csv"))
-    dataframe = pm4py.format_dataframe(dataframe, timest_format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    dataframe = pm4py.format_dataframe(dataframe, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
     log = log_converter.apply(dataframe, variant=log_converter.Variants.TO_EVENT_LOG, parameters={"stream_postprocessing": False})
     pm4py.write_xes(log, "non_postprocessed.xes")
     log = log_converter.apply(dataframe, variant=log_converter.Variants.TO_EVENT_LOG, parameters={"stream_postprocessing": True})

examples/dfg_min_ex_pandas.py

@@ -1,7 +1,6 @@
 import os
 
 import pm4py
-import pandas as pd
 from pm4py.util import constants, pandas_utils
 from pm4py.algo.discovery.dfg.adapters.pandas import df_statistics
 from pm4py.objects.conversion.dfg import converter as dfg_conv

examples/diagn_add_dataframe.py

@@ -1,7 +1,6 @@
 import pm4py
 from pm4py.algo.discovery.log_skeleton import algorithm as log_skeleton_discovery
 from pm4py.algo.conformance.log_skeleton import algorithm as log_skeleton_conformance
-import pandas as pd
 from pm4py.util import constants, pandas_utils
 
 

examples/events_distribution.py

@@ -1,16 +1,16 @@
 import os
 
-import pandas as pd
-
 from pm4py.util import constants, pandas_utils
+from pm4py.objects.log.util import dataframe_utils
 from pm4py.statistics.attributes.pandas import get as attr_get
 from pm4py.visualization.graphs import visualizer
 from examples import examples_conf
 
 
 def execute_script():
     df = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "receipt.csv"))
-    df["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(df["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
+
     # plots the distribution of the events over the days of a month
     x0, y0 = attr_get.get_events_distribution(df, distr_type="days_month")
     gviz = visualizer.apply(x0, y0, variant=visualizer.Variants.BARPLOT,

examples/execute_everything.py

@@ -937,7 +937,7 @@ def main():
     execute_script(link_analysis_vbfa)
     execute_script(ocel_streaming)
     execute_script(petri_manual_generation)
-    execute_script(timestamp_interleavings)
+    #execute_script(timestamp_interleavings)
     execute_script(object_centric_petri_net_discovery)
     execute_script(trans_system_stochastic_view)
     execute_script(network_analysis)

examples/feature_extraction_ocel.py

@@ -1,6 +1,5 @@
 import os
 import pm4py
-import pandas as pd
 from pm4py.algo.transformation.ocel.features.events import algorithm as events_feature_extraction
 from pm4py.util import pandas_utils, constants
 

examples/heuminer_plusplus.py

@@ -3,14 +3,13 @@
 from pm4py.visualization.heuristics_net import visualizer
 from pm4py.visualization.petri_net import visualizer as pn_visualizer
 from pm4py.util import constants, pandas_utils
+from pm4py.objects.log.util import dataframe_utils
 from examples import examples_conf
-import pandas as pd
 
 
 def execute_script():
     df = pandas_utils.read_csv("../tests/input_data/interval_event_log.csv")
-    df["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(df["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
-    df["start_timestamp"] = pandas_utils.dataframe_column_string_to_datetime(df["start_timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["start_timestamp", "time:timestamp"])
     log = pm4py.read_xes("../tests/input_data/interval_event_log.xes")
     heu_net = plusplus.apply_heu(log, parameters={"heu_net_decoration": "performance"})
     heu_net_2 = plusplus.apply_heu_pandas(df, parameters={"heu_net_decoration": "performance"})

examples/link_analysis_vbfa.py

@@ -1,13 +1,13 @@
-import pandas as pd
 from pm4py.util import constants, pandas_utils
 from pm4py.algo.discovery.ocel.link_analysis import algorithm as link_analysis
+from pm4py.objects.log.util import dataframe_utils
 import os
 
 
 def execute_script():
     dataframe = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "ocel", "VBFA.zip"), compression="zip", dtype="str")
     dataframe["time:timestamp"] = dataframe["ERDAT"] + " " + dataframe["ERZET"]
-    dataframe["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(dataframe["time:timestamp"], format="%Y%m%d %H%M%S", utc=constants.ENABLE_DATETIME_COLUMNS_AWARE)
+    dataframe = dataframe_utils.convert_timestamp_columns_in_df(dataframe, timest_format="%Y%m%d %H%M%S", timest_columns=["time:timestamp"])
     dataframe["RFWRT"] = dataframe["RFWRT"].astype(float)
     dataframe = link_analysis.apply(dataframe, parameters={"out_column": "VBELN", "in_column": "VBELV",
                                                            "sorting_column": "time:timestamp", "propagate": True})

examples/merging_case_relations.py

@@ -1,16 +1,16 @@
-import pandas as pd
 import pm4py
 from pm4py.util import constants, pandas_utils
+from pm4py.objects.log.util import dataframe_utils
 from pm4py.algo.merging.case_relations import algorithm as case_relations_merging
 from examples import examples_conf
 import os
 
 
 def execute_script():
     dataframe1 = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "receipt_even.csv"))
-    dataframe1["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(dataframe1["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
     dataframe2 = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "receipt_odd.csv"))
-    dataframe2["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(dataframe2["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    dataframe1 = dataframe_utils.convert_timestamp_columns_in_df(dataframe1, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
+    dataframe2 = dataframe_utils.convert_timestamp_columns_in_df(dataframe2, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     case_relations = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "case_relations.csv"))
     merged = case_relations_merging.apply(dataframe1, dataframe2, case_relations)
     dfg, sa, ea = pm4py.discover_dfg(merged)

examples/perf_spectrum_visualization.py

@@ -1,16 +1,16 @@
 import pm4py
 import os
 from pm4py.util import constants, pandas_utils
+from pm4py.objects.log.util import dataframe_utils
 from examples import examples_conf
-import pandas as pd
 
 
 def execute_script():
     log = pm4py.read_xes(os.path.join("..", "tests", "input_data", "receipt.xes"))
     pm4py.view_performance_spectrum(log, ["Confirmation of receipt", "T04 Determine confirmation of receipt",
                                          "T10 Determine necessity to stop indication"], format=examples_conf.TARGET_IMG_FORMAT)
     df = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "receipt.csv"))
-    df["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(df["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     pm4py.view_performance_spectrum(df, ["Confirmation of receipt", "T04 Determine confirmation of receipt",
                                          "T10 Determine necessity to stop indication"], format=examples_conf.TARGET_IMG_FORMAT)
 

examples/resource_profiles_pandas.py

@@ -1,13 +1,12 @@
-import pandas as pd
-import pm4py
 from pm4py.util import pandas_utils, constants
+from pm4py.objects.log.util import dataframe_utils
 import os
 from pm4py.algo.organizational_mining.resource_profiles import algorithm
 
 
 def execute_script():
     log = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "running-example.csv"))
-    log["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(log["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    log = dataframe_utils.convert_timestamp_columns_in_df(log, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     # Metric RBI 1.1: Number of distinct activities done by a resource in a given time interval [t1, t2)
     print(algorithm.distinct_activities(log, "2010-12-30 00:00:00", "2011-01-25 00:00:00", "Sara"))
     # Metric RBI 1.3: Fraction of completions of a given activity a, by a given resource r,

examples/simplified_interface.py

@@ -2,7 +2,7 @@
 
 import pm4py
 from pm4py.util import constants, pandas_utils
-import pandas as pd
+from pm4py.objects.log.util import dataframe_utils
 from examples import examples_conf
 
 
@@ -14,7 +14,7 @@ def execute_script():
 
     # reads a CSV into a dataframe
     df = pandas_utils.read_csv("../tests/input_data/running-example.csv")
-    df["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(df["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    df = dataframe_utils.convert_timestamp_columns_in_df(df, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     df["case:concept:name"] = df["case:concept:name"].astype("string")
 
     # converts the dataframe to an event log

examples/temporal_profile_dataframe.py

@@ -1,4 +1,3 @@
-import pandas as pd
 from pm4py.algo.discovery.temporal_profile import algorithm as temporal_profile_discovery
 from pm4py.algo.conformance.temporal_profile import algorithm as temporal_profile_conformance
 from pm4py.util import constants, pandas_utils
@@ -7,7 +6,7 @@
 
 def execute_script():
     dataframe = pandas_utils.read_csv("../tests/input_data/receipt.csv")
-    dataframe = dataframe_utils.convert_timestamp_columns_in_df(dataframe, timest_format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    dataframe = dataframe_utils.convert_timestamp_columns_in_df(dataframe, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
     tf = temporal_profile_discovery.apply(dataframe)
     conformance = temporal_profile_conformance.apply(dataframe, tf, parameters={"zeta": 6.0})
     for index, dev in enumerate(conformance):

examples/timestamp_granularity.py

@@ -1,12 +1,11 @@
-import pandas as pd
 import pm4py
 from pm4py.util import constants, pandas_utils
 import time
 
 
 def execute_script():
     dataframe = pandas_utils.read_csv("../tests/input_data/receipt.csv")
-    dataframe = pm4py.format_dataframe(dataframe, timest_format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    dataframe = pm4py.format_dataframe(dataframe, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
 
     # prints the original timestamp column of the dataframe
     print(dataframe["time:timestamp"])

examples/timestamp_interleavings.py

@@ -1,16 +1,16 @@
 from pm4py.util import constants, pandas_utils
-import pandas as pd
 from pm4py.algo.discovery.ocel.interleavings import algorithm as interleavings_miner
 from pm4py.visualization.ocel.interleavings import visualizer as interleavings_visualizer
+from pm4py.objects.log.util import dataframe_utils
 from examples import examples_conf
 import os
 
 
 def execute_script():
     receipt_even = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "receipt_even.csv"))
-    receipt_even["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(receipt_even["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
     receipt_odd = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "receipt_odd.csv"))
-    receipt_odd["time:timestamp"] = pandas_utils.dataframe_column_string_to_datetime(receipt_odd["time:timestamp"], utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_XES_TIMESTAMP_PARSE_FORMAT)
+    receipt_even = dataframe_utils.convert_timestamp_columns_in_df(receipt_even, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
+    receipt_odd = dataframe_utils.convert_timestamp_columns_in_df(receipt_odd, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT, timest_columns=["time:timestamp"])
     case_relations = pandas_utils.read_csv(os.path.join("..", "tests", "input_data", "interleavings", "case_relations.csv"))
     interleavings_dataframe = interleavings_miner.apply(receipt_even, receipt_odd, case_relations)
     print(interleavings_dataframe)

examples/transform_db_to_ocel.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 import pm4py
 from pm4py.objects.ocel.obj import OCEL
-import pandas as pd
 from pm4py.util import pandas_utils, constants
 from pm4py.util.dt_parsing.variants import strpfromiso
 import os

pm4py/algo/filtering/pandas/timestamp/timestamp_filter.py

@@ -43,8 +43,7 @@ def filter_traces_contained(df: pd.DataFrame, dt1: Union[str, datetime.datetime]
     case_id_glue = exec_utils.get_param_value(Parameters.CASE_ID_KEY, parameters, CASE_CONCEPT_NAME)
     dt1 = get_dt_from_string(dt1)
     dt2 = get_dt_from_string(dt2)
-    dt1 = pandas_utils.dataframe_column_string_to_datetime(dt1, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
-    dt2 = pandas_utils.dataframe_column_string_to_datetime(dt2, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
+
     grouped_df = df[[case_id_glue, timestamp_key]].groupby(df[case_id_glue])
     first = grouped_df.first()
     last = grouped_df.last()
@@ -88,8 +87,6 @@ def filter_traces_intersecting(df: pd.DataFrame, dt1: Union[str, datetime.dateti
     case_id_glue = exec_utils.get_param_value(Parameters.CASE_ID_KEY, parameters, CASE_CONCEPT_NAME)
     dt1 = get_dt_from_string(dt1)
     dt2 = get_dt_from_string(dt2)
-    dt1 = pandas_utils.dataframe_column_string_to_datetime(dt1, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
-    dt2 = pandas_utils.dataframe_column_string_to_datetime(dt2, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
     grouped_df = df[[case_id_glue, timestamp_key]].groupby(df[case_id_glue])
     first = grouped_df.first()
     last = grouped_df.last()
@@ -137,8 +134,6 @@ def apply_events(df: pd.DataFrame, dt1: Union[str, datetime.datetime], dt2: Unio
     timestamp_key = exec_utils.get_param_value(Parameters.TIMESTAMP_KEY, parameters, DEFAULT_TIMESTAMP_KEY)
     dt1 = get_dt_from_string(dt1)
     dt2 = get_dt_from_string(dt2)
-    dt1 = pandas_utils.dataframe_column_string_to_datetime(dt1, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
-    dt2 = pandas_utils.dataframe_column_string_to_datetime(dt2, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
 
     ret = df[df[timestamp_key] >= dt1]
     ret = ret[ret[timestamp_key] <= dt2]
@@ -180,8 +175,6 @@ def filter_traces_attribute_in_timeframe(df: pd.DataFrame, attribute: str, attri
 
     dt1 = get_dt_from_string(dt1)
     dt2 = get_dt_from_string(dt2)
-    dt1 = pandas_utils.dataframe_column_string_to_datetime(dt1, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
-    dt2 = pandas_utils.dataframe_column_string_to_datetime(dt2, utc=constants.ENABLE_DATETIME_COLUMNS_AWARE, format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
 
     filtered = df[df[attribute] == attribute_value]
     filtered = filtered[filtered[timestamp_key] >= dt1]

pm4py/objects/conversion/log/variants/to_data_frame.py

@@ -47,8 +47,7 @@ def apply(log, parameters=None):
         log = to_event_stream.apply(log, parameters=new_parameters)
 
     transf_log = [dict(x) for x in log]
-
-    df = pandas_utils.instantiate_dataframe_from_dict(transf_log)
+    df = pandas_utils.instantiate_dataframe(transf_log)
 
     df.attrs = copy(log.properties)
     if pm4_constants.PARAMETER_CONSTANT_CASEID_KEY in df.attrs:

pm4py/objects/log/importer/xes/variants/rustxes.py

@@ -5,6 +5,8 @@
 from pm4py.objects.conversion.log import converter as log_converter
 import pandas as pd
 from copy import copy
+from pm4py.util.dt_parsing.variants import strpfromiso
+import importlib.util
 
 
 class Parameters(Enum):
@@ -20,7 +22,17 @@ def apply(log_path: str, parameters: Optional[Dict[Any, Any]] = None) -> Union[E
     import rustxes
 
     log = rustxes.import_xes(log_path)
-    log = log[0].to_pandas()
+    log = log[0]
+    log = log.to_pandas()
+
+    for col in log.columns:
+        if "date" in str(log[col].dtype) or "time" in str(log[col].dtype):
+            log[col] = strpfromiso.fix_dataframe_column(log[col])
+
+    if importlib.util.find_spec("cudf"):
+        import cudf
+        log = cudf.DataFrame(log)
+        return log
 
     if return_legacy_log_object:
         this_parameters = copy(parameters)