Skip to content

Commit

Permalink
v2024.07.24
Browse files Browse the repository at this point in the history 
nullable int fix
  • Loading branch information
@Beercow
Beercow committed Jul 24, 2024
1 parent 2fb5195 commit 8296858
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 77 deletions.
Binary file modified OneDriveExplorer/Images/splashv.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion OneDriveExplorer/OneDriveExplorer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
)

__author__ = "Brian Maloney"
__version__ = "2024.07.19"
__version__ = "2024.07.24"
__email__ = "bmmaloney97@gmail.com"
rbin = []
DATParser = dat_parser.DATParser()
Expand Down
4 changes: 2 additions & 2 deletions OneDriveExplorer/OneDriveExplorer_GUI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
)

__author__ = "Brian Maloney"
__version__ = "2024.07.19"
__version__ = "2024.07.24"
__email__ = "bmmaloney97@gmail.com"
rbin = []
user_logs = {}
Expand Down Expand Up @@ -203,7 +203,7 @@ def setup_window_style(self):
set_window_long(hwnd, GWL_STYLE, new_style)

def create_widgets(self):
self.frame = ttk.Frame(self.win, relief='groove')
self.frame = ttk.Frame(self.win, relief='flat')
self.inner_frame = ttk.Frame(self.frame, relief='groove', padding=5)

self.frame.grid(row=0, column=0)
Expand Down
4 changes: 3 additions & 1 deletion OneDriveExplorer/ode/parsers/csv_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ def parse_csv(filename):
'Media': 'object',
'parentScopeID': 'object',
'folderStatus': 'Int64',
'Path': 'object'
'Path': 'object',
'shortcutVolumeID': 'Int64',
'shortcutItemIndex': 'Int64'
}

try:
Expand Down
33 changes: 11 additions & 22 deletions OneDriveExplorer/ode/parsers/dat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from dissect import cstruct
import pandas as pd
import numpy as np
from ode.utils import progress, progress_gui, permissions
from ode.utils import progress, progress_gui, permissions, change_dtype

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -392,7 +392,7 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
continue
block._values.update([('siteID', b''), ('webID', b'')])
block._values.move_to_end('listID', last=True)
block._values.update([('libraryType', b''), ('spoPermissions', ''), ('shortcutVolumeID', ''), ('shortcutItemIndex', '')])
block._values.update([('libraryType', ''), ('spoPermissions', ''), ('shortcutVolumeID', ''), ('shortcutItemIndex', '')])

elif ff == '0b':
data_type = 'Scope'
Expand All @@ -402,7 +402,7 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
del block._values[key]
except Exception:
continue
block._values.update([('siteID', b''), ('webID', b''), ('listID', b''), ('libraryType', b''), ('spoPermissions', '')])
block._values.update([('siteID', b''), ('webID', b''), ('listID', b''), ('libraryType', ''), ('spoPermissions', '')])
block._values.move_to_end('shortcutVolumeID', last=True)
block._values.move_to_end('shortcutItemIndex', last=True)

Expand Down Expand Up @@ -479,56 +479,45 @@ def parse_dat(self, usercid, account='Business', gui=False, pb=False, value_labe
temp_files.seek(0)
temp_folders.seek(0)

convert = {'shortcutVolumeID': 'Int64',
'shortcutItemIndex': 'Int64'
}

df_scope = pd.read_csv(temp_scope)
temp_scope.close()
df_scope.insert(0, 'Type', 'Scope')
df_scope.insert(5, 'tenantID', '')
df_scope.insert(6, 'webURL', '')
df_scope.insert(7, 'remotePath', '')
df_scope = df_scope.astype(object)
df_scope = df_scope.astype(convert)
df_scope['shortcutVolumeID'].fillna(0, inplace=True)
df_scope['shortcutItemIndex'].fillna(0, inplace=True)

df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')
df_scope['spoPermissions'].replace('', np.nan, inplace=True)
df_scope['spoPermissions'] = df_scope['spoPermissions'].fillna(0).astype('int')
df_scope = change_dtype(df_scope, df_name='df_scope')
df_scope['spoPermissions'] = df_scope['spoPermissions'].apply(lambda x: permissions(x))
df_scope.fillna('', inplace=True)
scopeID = df_scope['scopeID'].tolist()

df_files = pd.read_csv(temp_files, usecols=['parentResourceID', 'resourceID', 'eTag', 'fileName', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'size', 'localHashDigest', 'sharedItem', 'mediaDateTaken', 'mediaWidth', 'mediaHeight', 'mediaDuration'])
temp_files.close()
df_files['HydrationTime'] = ''
df_files['localHashAlgorithm'] = 0
df_files.insert(0, 'Type', 'File')
df_files.rename(columns={"fileName": "Name",
"mediaDateTaken": "DateTaken",
"mediaWidth": "Width",
"mediaHeight": "Height",
"mediaDuration": "Duration"
}, inplace=True)
df_files['DateTaken'] = pd.to_datetime(df_files['DateTaken'], unit='s').astype(str)
df_files = change_dtype(df_files, df_name='df_files')
columns = ['DateTaken', 'Width', 'Height', 'Duration']
df_files['Media'] = df_files[columns].to_dict(orient='records')
df_files = df_files.drop(columns=columns)
if account == 'Personal':
df_files['localHashDigest'] = df_files['localHashDigest'].apply(lambda x: f'SHA1({x})')
else:
df_files['localHashDigest'] = df_files['localHashDigest'].apply(lambda x: f'quickXor({codecs.encode(binascii.unhexlify(x), "base64").decode("utf-8").rstrip()})')
df_files['size'] = df_files['size'].apply(lambda x: f'{x//1024 + 1:,} KB')
df_files['size'] = df_files['size'].apply(lambda x: '0 KB' if x == 0 else f'{x//1024 + 1:,} KB')
df_files['spoPermissions'] = df_files['spoPermissions'].apply(lambda x: permissions(x))
df_files['lastChange'] = pd.to_datetime(df_files['lastChange'], unit='s').astype(str)
df_files.insert(0, 'Type', 'File')

df_folders = pd.read_csv(temp_folders, usecols=['parentScopeID', 'parentResourceID', 'resourceID', 'eTag', 'folderName', 'folderStatus', 'spoPermissions', 'volumeID', 'itemIndex'])
temp_folders.close()
df_folders.insert(0, 'Type', 'Folder')
df_folders.rename(columns={"folderName": "Name"}, inplace=True)
df_folders = change_dtype(df_folders, df_name='df_folders')
df_folders['spoPermissions'] = df_folders['spoPermissions'].apply(lambda x: permissions(x))
df_folders.insert(0, 'Type', 'Folder')

df = pd.concat([df_scope, df_files, df_folders], ignore_index=True, axis=0)
df = df.where(pd.notnull(df), None)

Expand Down
44 changes: 34 additions & 10 deletions OneDriveExplorer/ode/parsers/onedrive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ def find_parent(self, x, id_name_dict, parent_dict):

# Generate scopeID list instead of passing
def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_path, rbin_df, account=False, reghive=False, recbin=False, gui=False, pb=False, value_label=False):

df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
df_scope['shortcutVolumeID'] = df_scope['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')

if os.path.isdir(file_path):
directory = file_path
filename = ['SyncEngineDatabase.db', 'SafeDelete.db']
Expand Down Expand Up @@ -134,7 +138,9 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
convert = {'fileStatus': 'Int64',
'volumeID': 'Int64',
'sharedItem': 'Int64',
'folderStatus': 'Int64'
'folderStatus': 'Int64',
'shortcutVolumeID': 'Int64',
'shortcutItemIndex': 'Int64'
}

else:
Expand All @@ -145,7 +151,9 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
'volumeID': 'Int64',
'itemIndex': 'Int64',
'sharedItem': 'Int64',
'folderStatus': 'Int64'
'folderStatus': 'Int64',
'shortcutVolumeID': 'Int64',
'shortcutItemIndex': 'Int64'
}

parent_resource_dict = df[(df['resourceID'].notnull()) & (df['Type'] == 'Folder')].set_index('resourceID').apply(lambda x: x['Path'] + '\\' + x['Name'], axis=1).to_dict()
Expand All @@ -169,33 +177,45 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
df = df.astype(convert)
df['volumeID'].fillna(0, inplace=True)
df['itemIndex'].fillna(0, inplace=True)
df['shortcutVolumeID'].fillna(0, inplace=True)
df['shortcutItemIndex'].fillna(0, inplace=True)

df['volumeID'] = df['volumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
df['volumeID'] = df['volumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')
df['shortcutVolumeID'] = df['shortcutVolumeID'].apply(lambda x: '{:08x}'.format(x) if pd.notna(x) else '')
df['shortcutVolumeID'] = df['shortcutVolumeID'].apply(lambda x: '{}{}{}{}-{}{}{}{}'.format(*x.upper()) if x else '')


cache = {}
final = []
dcache = {}
is_del = []

# Need to look into this
if not df_GraphMetadata_Records.empty:
df_GraphMetadata_Records.set_index('resourceID', inplace=True)

column_len = len(df.columns)

for row in df.sort_values(
by=['Level', 'parentResourceID', 'Type', 'FileSort', 'FolderSort', 'libraryType'],
ascending=[False, False, False, True, False, False]).to_dict('records'):
if row['Type'] == 'File':
if column_len == 32:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'size', 'localHashDigest', 'sharedItem', 'Media')}
try:
if column_len == 32:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'size', 'localHashDigest', 'sharedItem', 'Media')}

if column_len == 33:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'HydrationTime', 'size', 'localHashDigest', 'sharedItem', 'Media')}
if column_len == 33:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'HydrationTime', 'size', 'localHashDigest', 'sharedItem', 'Media')}

if column_len == 36:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'lastHydrationType', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'firstHydrationTime', 'lastHydrationTime', 'hydrationCount', 'size', 'localHashDigest', 'sharedItem', 'Media')}
if column_len == 36:
file = {key: row[key] for key in ('parentResourceID', 'resourceID', 'eTag', 'Path', 'Name', 'fileStatus', 'lastHydrationType', 'spoPermissions', 'volumeID', 'itemIndex', 'lastChange', 'firstHydrationTime', 'lastHydrationTime', 'hydrationCount', 'size', 'localHashDigest', 'sharedItem', 'Media')}

except Exception as e:
if gui:
log.error(f'Unable to read dataframe. Something went wrong. {e}')
else:
print(f'Unable to read dataframe. Something went wrong. {e}')
return {}, rbin_df

file.setdefault('Metadata', '')

Expand Down Expand Up @@ -269,5 +289,9 @@ def parse_onedrive(self, df, df_scope, df_GraphMetadata_Records, scopeID, file_p
cache['Data'] = final

df_GraphMetadata_Records.reset_index(inplace=True)
try:
df_GraphMetadata_Records.drop('index', axis=1, inplace=True)
except:
pass

return cache, rbin_df
Loading

0 comments on commit 8296858

Please sign in to comment.