Skip to content

Commit

Permalink
api: set columns min/max and stats when columns updated
Browse files Browse the repository at this point in the history
  • Loading branch information
@tschaume
tschaume committed Dec 4, 2021
1 parent 1e43b34 commit 89aa466
Showing 1 changed file with 30 additions and 92 deletions.
122 changes: 30 additions & 92 deletions mpcontribs-api/mpcontribs/api/projects/document.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,9 @@ def post_save(cls, sender, document, **kwargs):
TopicArn=resp["TopicArn"], Protocol="email", Endpoint=endpoint
)
else:
set_keys = document._delta()[0].keys()
delta_set, delta_unset = document._delta()

if "is_approved" in set_keys and document.is_approved:
if "is_approved" in delta_set and document.is_approved:
subject = f'Your project "{document.name}" has been approved'
netloc = urllib.parse.urlparse(request.url).netloc.replace("-api", "")
portal = f"{scheme}://{netloc}"
Expand All @@ -212,62 +212,47 @@ def post_save(cls, sender, document, **kwargs):
)
send_email(topic_arn, subject, html)

if "columns" in set_keys:
if "columns" in delta_set or "columns" in delta_unset:
# document.columns has been updated as intended by the user
from mpcontribs.api.contributions.document import Contributions, COMPONENTS

columns = {col.path: col for col in document.columns}
min_max_paths = []

# set columns field for project
if is_quantity or is_text or isinstance(value, bool):
if path not in columns:
columns[path] = Column(path=path)

if is_quantity:
columns[path].unit = value["unit"]

if is_quantity:
min_max_paths.append(path)

ncolumns = len(columns)
if ncolumns > MAX_COLUMNS:
raise ValueError(f"Reached maximum number of columns ({MAX_COLUMNS})!")

ncolumns = len(document.columns)
if ncolumns > MAX_COLUMNS:
raise ValueError(f"Reached maximum number of columns ({MAX_COLUMNS})!")

# get and set min/max for all paths
group = {"_id": None}
# set min/max for all number columns
columns = {col.path: col for col in document.columns}

for path in paths:
field = f"{path}{delimiter}value"
for k in ["min", "max"]:
clean_path = path.replace(delimiter, "__")
key = f"{clean_path}__{k}"
group[key] = {f"${k}": f"${field}"}
if document.columns:
min_max_paths = [col["path"] for col in document.columns if col["unit"] != "NaN"]
group = {"_id": None}

pipeline = [{"$match": {"project": project_name}}, {"$group": group}]
result = list(Contributions.objects.aggregate(pipeline))
min_max = None if not result else result[0]
for path in min_max_paths:
field = f"{path}{delimiter}value"
for k in ["min", "max"]:
clean_path = path.replace(delimiter, "__")
key = f"{clean_path}__{k}"
group[key] = {f"${k}": f"${field}"}

for clean_path in min_max_paths:
for k in ["min", "max"]:
path = clean_path.replace(delimiter, "__")
m = min_max.get(f"{path}__{k}")
if m is not None:
setattr(columns[clean_path], k, m)
pipeline = [{"$match": {"project": document.id}}, {"$group": group}]
result = list(Contributions.objects.aggregate(pipeline))
min_max = {} if not result else result[0]

# add/remove columns for other components
for path in COMPONENTS.keys():
if path not in columns and getattr(document, path):
columns[path] = Column(path=path)
for clean_path in min_max_paths:
for k in ["min", "max"]:
path = clean_path.replace(delimiter, "__")
m = min_max.get(f"{path}__{k}")
if m is not None:
setattr(columns[clean_path], k, m)

# update stats
ncontribs = Contributions.objects(project=project.name).count()
ncontribs = Contributions.objects(project=document.id).count()
stats_kwargs = {"columns": len(columns), "contributions": ncontribs}

for component in COMPONENTS.keys():
pipeline = [
{"$match": {
"project": project.name,
"project": document.id,
component: {
"$exists": True,
"$not": {"$size": 0}
Expand All @@ -279,7 +264,7 @@ def post_save(cls, sender, document, **kwargs):
stats_kwargs[component] = result[0]["count"] if result else 0

stats = Stats(**stats_kwargs)
document.update(stats=stats)
document.update(stats=stats, columns=columns.values())

@classmethod
def post_delete(cls, sender, document, **kwargs):
Expand All @@ -296,53 +281,6 @@ def post_delete(cls, sender, document, **kwargs):
send_email(topic_arn, subject, html)
sns_client.delete_topic(TopicArn=topic_arn)

# TODO from contributions signals
@classmethod
def post_delete(cls, sender, document, **kwargs):
if kwargs.get("skip"):
return

remaining_time = kwargs.get("remaining_time")
if remaining_time is not None and remaining_time < 5:
print("NOT ENOUGH TIME LEFT FOR POST_DELETE!")
return

# reset columns field for project
project = document.project.fetch().reload("columns")
columns_copy = deepcopy(project.columns)
columns = {col.path: col for col in project.columns}
min_max_paths = []

for path in list(columns.keys()):
column = columns[path]

if not isnan(column.min) and not isnan(column.max):
min_max_paths.append(path)
else:
result = list(sender.objects.aggregate([
{"$match": {"project": project.name, path: {"$exists": True}}},
{"$count": "count"}
]))
if result and result[0]["count"] < 1:
columns.pop(path)

# get and set min/max for all paths
min_max = get_min_max(sender, min_max_paths, project.name)

for clean_path in min_max_paths:
path = clean_path.replace(delimiter, "__")
column = columns[clean_path]

for k in ["min", "max"]:
if min_max.get(f"{path}__{k}") is None:
# just deleted last contribution with this column
columns.pop(clean_path)
break

cls.update_project(sender, project, columns_copy, columns.values())




register_field(ProviderEmailField, ProviderEmail, available_params=(params.LengthParam,))
signals.post_save.connect(Projects.post_save, sender=Projects)
Expand Down

0 comments on commit 89aa466

Please sign in to comment.