-
Notifications
You must be signed in to change notification settings - Fork 0
/
hltb_barchartrace.py
193 lines (157 loc) · 6.32 KB
/
hltb_barchartrace.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import glob
import importlib
import sys
import pandas as pd
# Import functions from hltb_sanitizer
sanitizer_module = importlib.import_module("hltb_sanitizer")
# Tags to exclude from results, possible to use multiple tags, exmaple: ["Backlog", "Retired"]
BLOCK_TAGS = ["Blocked"]
# Custom tab names
CUSTOM_TAGS = ["Stalled"]
# Preferred finished date, accepted values: "Finished", "Lastmod"
DATE_COL = "Finished"
# Accepted vlaues: "Platform", "Storefront"
DIVISION = "Platform"
# Organize sanitized CSV
def sort_data(df):
# Define the sort keys in the desired order
sort_keys = ["Date", DATE_COL, "Platform", "Storefront", "Title"]
# Sort the data based on the sort keys
df_sorted = df.sort_values(by=sort_keys, na_position="last")
# Reset the index of the sorted DataFrame
df_sorted = df_sorted.reset_index(drop=True)
# Return the sorted DataFrame
return df_sorted[sort_keys]
def calculate_number(df, division):
# Sort the DataFrame by 'Date' column in ascending order
df_sorted = df.sort_values(by="Date")
# Initialize an empty list to store the calculated 'Number' values
number_values = []
# Iterate over each row in the sorted DataFrame
for _index, row in df_sorted.iterrows():
# Get the current 'Date' and 'Platform'/'Storefront' values
current_date = row["Date"]
current_platform = row[division]
# Count occurrences of current 'Platform'/'Storefront' in the rows with dates to date
count = (
df_sorted.loc[df_sorted["Date"] <= current_date]
.loc[df_sorted[division] == current_platform]
.shape[0]
)
# Append the count to the list of 'Number' values
number_values.append(count)
# Add the 'Number' column to the DataFrame
df_sorted["Number"] = number_values
# Create a new DataFrame with only the 'Date', 'Platform'/'Storefront', and 'Number' columns
df_sorted = df_sorted[[division, "Date", "Number"]]
# Drop the duplicate rows
df_sorted = df_sorted.drop_duplicates()
# Filter out rows where 'Number' is 0
df_sorted = df_sorted[df_sorted["Number"] != 0]
# Filter out rows where 'Date' is later than '2022-10-31'
# df_sorted = df_sorted[df_sorted['Date'] <= '2022-10-31']
# Create a new DataFrame with all unique 'Date' and 'Platform'/'Storefront' combinations
unique_dates = df_sorted["Date"].unique()
unique_platforms = df_sorted[division].unique()
new_index = pd.MultiIndex.from_product(
[unique_dates, unique_platforms], names=["Date", division]
)
# new_df = df_sorted.set_index(['Date', division]).reindex(new_index)
new_df = pd.DataFrame(index=new_index).reset_index()
# Merge the new DataFrame with the sorted DataFrame
merged_df = pd.merge(new_df, df_sorted, on=["Date", division], how="left")
# Forward fill the missing values within each group of same division values
merged_df["Number"] = merged_df.groupby(division)["Number"].ffill()
# Fill the first 'Number' value of every division values with 0
merged_df["Number"] = merged_df.groupby(division)["Number"].transform(
lambda x: x.fillna(0)
)
return merged_df
def sanitized_dataframe_post(df, division):
# Use shorter alias for platform/storefront
if division == "Platform":
df["Platform"] = df["Platform"].replace(
{
"NES": "FC",
"Super Nintendo": "SFC",
"Nintendo DS": "NDS",
"Nintendo 3DS": "3DS",
"Nintendo 64": "N64",
"Nintendo GameCube": "NGC",
"Nintendo Switch": "Switch",
"Game Boy": "GB",
"Game Boy Color": "GBC",
"Game Boy Advance": "GBA",
"Xbox 360": "X360",
"Xbox Series X/S": "XSS",
"PlayStation VR": "PSVR",
"PlayStation Vita": "PSV",
"PlayStation Portable": "PSP",
"PlayStation 5": "PS5",
"PlayStation 4": "PS4",
"PlayStation 3": "PS3",
"PlayStation 2": "PS2",
"PlayStation 1": "PS1",
"Sega Master System": "SMS",
"Sega Mega Drive/Genesis": "MD",
"Sega Saturn": "SS",
"Sega Game Gear": "GG",
"Neo Geo Pocket": "NGPC",
"WonderSwan": "WSC",
"NEC PC-98": "PC-98",
"TurboGrafx-16": "PCE",
"TurboGrafx-CD": "PCE-CD",
"Oculus Quest": "Meta Quest",
}
)
elif division == "Storefront":
df["Storefront"] = df["Storefront"].replace(
{
"Direct Download": "DL",
"Xbox Game Pass": "XGP",
"Xbox Games w/ Gold": "XGP",
"Xbox Store": "Xbox",
"Microsoft Store": "Microsoft",
"Ubisoft Connect": "Ubisoft",
"Nintendo eShop": "eShop",
"Google Play Pass": "Play Pass",
"Epic Games": "EGS",
"PlayStation Plus": "PS+",
"PlayStation Store": "PSN",
"itch.io": "itch",
}
)
else:
print("Invalid division. Exiting.")
sys.exit()
return df
# File naming scheme
file_list = glob.glob("HLTB_Games_*.csv")
# Skip problematic lines
skip_rows = []
# Read CSV file
if len(file_list) > 0:
# Sanitize every file
for filepath in file_list:
new_file_name = filepath.replace(
"HLTB_Games_", "HLTB-barchartrace-by-" + DIVISION.lower() + "-"
)
df_raw = pd.read_csv(filepath, skiprows=skip_rows)
df_mod = sanitizer_module.sanitized_dataframe(df_raw)
df_mod = sanitizer_module.date_sanitize(df_mod)
# Sort data
df_mod = sort_data(df_mod)
# Calculate number of platforms at a specific date
df_mod = calculate_number(df_mod, DIVISION)
# Post sanitization
df_mod = sanitized_dataframe_post(df_mod, DIVISION)
# Debug preview
print(df_mod.head())
# Export to CSV
df_mod.to_csv(new_file_name, index=False, quoting=1)
print("Now drop output to https://fabdevgit.github.io/barchartrace")
else:
print("HLTB sanitized CSV not found.")
sys.exit()