Skip to content

Commit

Permalink
Merge branch 'master' into argoai
Browse files Browse the repository at this point in the history
  • Loading branch information
RishabhJain2018 authored Feb 5, 2020
2 parents 8d52591 + c84316a commit 96968d6
Show file tree
Hide file tree
Showing 45 changed files with 1,704 additions and 149 deletions.
24 changes: 15 additions & 9 deletions apps/base/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def get_boto3_client(resource, aws_keys):
logger.exception(e)


def get_sqs_queue_object():
def get_or_create_sqs_queue_object(queue_name):
if settings.DEBUG or settings.TEST:
queue_name = "evalai_submission_queue"
sqs = boto3.resource(
Expand Down Expand Up @@ -212,19 +212,25 @@ def send_slack_notification(webhook=settings.SLACK_WEB_HOOK_URL, message=""):
try:
data = {
"text": message["text"],
"attachments": [
{
"color": "ffaf4b",
"fields": message["fields"]
}
]
"attachments": [{"color": "ffaf4b", "fields": message["fields"]}],
}
return requests.post(
webhook,
data=json.dumps(data),
headers={"Content-Type": "application/json"}
headers={"Content-Type": "application/json"},
)
except Exception as e:
logger.exception(
"Exception raised while sending slack notification. \n Exception message: {}".format(e)
"Exception raised while sending slack notification. \n Exception message: {}".format(
e
)
)


def mock_if_non_prod_aws(aws_mocker):
def decorator(func):
if not (settings.DEBUG or settings.TEST):
return func
return aws_mocker(func)

return decorator
4 changes: 4 additions & 0 deletions apps/challenges/admin.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from django import forms
from django.contrib import admin, messages

from django.contrib.admin.helpers import ActionForm

from base.admin import ImportExportTimeStampedAdmin
Expand All @@ -12,6 +13,8 @@
stop_workers,
)

from .admin_filters import ChallengeFilter

from .models import (
Challenge,
ChallengeConfiguration,
Expand Down Expand Up @@ -52,6 +55,7 @@ class ChallengeAdmin(ImportExportTimeStampedAdmin):
"task_def_arn",
)
list_filter = (
ChallengeFilter,
"published",
"is_registration_open",
"enable_forum",
Expand Down
38 changes: 38 additions & 0 deletions apps/challenges/admin_filters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
from django.contrib.admin import SimpleListFilter
from django.utils import timezone


class ChallengeFilter(SimpleListFilter):

title = "Challenges"
parameter_name = "challenge"

def lookups(self, request, model_admin):
options = [
("past", "Past"),
("present", "Ongoing"),
("future", "Upcoming"),
]
return options

def queryset(self, request, queryset):
q_params = {
"published": True,
"approved_by_admin": True,
"is_disabled": False,
}
if self.value() == "past":
q_params["end_date__lt"] = timezone.now()
challenges = queryset.filter(**q_params)
return challenges

elif self.value() == "present":
q_params["start_date__lt"] = timezone.now()
q_params["end_date__gt"] = timezone.now()
challenges = queryset.filter(**q_params)
return challenges

elif self.value() == "future":
q_params["start_date__gt"] = timezone.now()
challenges = queryset.filter(**q_params)
return challenges
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.20 on 2019-07-17 09:25
from __future__ import unicode_literals

import django.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('challenges', '0058_add_show_leaderboard_by_latest_submission_field_in_challenge_phase_split_model'),
]

operations = [
migrations.AddField(
model_name='challengephase',
name='environment_url',
field=models.CharField(max_length=2128, null=True, validators=[django.core.validators.URLValidator()]),
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-12-24 17:28
from __future__ import unicode_literals

import django.core.validators
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("challenges", "0058_adding_environment_url_for_rl_challenges")
]

operations = [
migrations.AlterField(
model_name="challengephase",
name="environment_url",
field=models.CharField(
blank=True,
max_length=2128,
null=True,
validators=[django.core.validators.URLValidator()],
),
)
]
22 changes: 15 additions & 7 deletions apps/challenges/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.core.validators import URLValidator
from django.db.models.signals import pre_save
from django.dispatch import receiver
from django.utils import timezone
Expand Down Expand Up @@ -84,7 +85,7 @@ def __init__(self, *args, **kwargs):
models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True
null=True,
)
remote_evaluation = models.BooleanField(
default=False, verbose_name="Remote Evaluation", db_index=True
Expand Down Expand Up @@ -122,11 +123,11 @@ def __init__(self, *args, **kwargs):
max_length=20, verbose_name="evalai-cli version", null=True, blank=True
)
# The number of active workers on Fargate of the challenge.
workers = models.IntegerField(
null=True, blank=True, default=None
)
workers = models.IntegerField(null=True, blank=True, default=None)
# The task definition ARN for the challenge, used for updating and creating service.
task_def_arn = models.CharField(null=True, blank=True, max_length=2048, default="")
task_def_arn = models.CharField(
null=True, blank=True, max_length=2048, default=""
)

class Meta:
app_label = "challenges"
Expand Down Expand Up @@ -170,7 +171,9 @@ def is_active(self):
weak=False,
)
signals.post_save.connect(
model_field_name(field_name="evaluation_script")(restart_workers_signal_callback),
model_field_name(field_name="evaluation_script")(
restart_workers_signal_callback
),
sender=Challenge,
weak=False,
)
Expand Down Expand Up @@ -232,6 +235,9 @@ def __init__(self, *args, **kwargs):
null=True,
)
slug = models.SlugField(max_length=200, null=True, unique=True)
environment_url = models.CharField(
validators=[URLValidator()], null=True, blank=True, max_length=2128
) # Max length of URL and tag is 2000 and 128 respectively

class Meta:
app_label = "challenges"
Expand Down Expand Up @@ -280,7 +286,9 @@ def save(self, *args, **kwargs):
weak=False,
)
signals.post_save.connect(
model_field_name(field_name="test_annotation")(restart_workers_signal_callback),
model_field_name(field_name="test_annotation")(
restart_workers_signal_callback
),
sender=ChallengePhase,
weak=False,
)
Expand Down
1 change: 1 addition & 0 deletions apps/challenges/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ class Meta:
"challenge_phase_name",
"dataset_split_name",
"visibility",
"show_leaderboard_by_latest_submission"
)

def get_dataset_split_name(self, obj):
Expand Down
5 changes: 5 additions & 0 deletions apps/challenges/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,4 +151,9 @@
views.get_challenge_phase_by_slug,
name="get_challenge_phase_by_slug",
),
url(
r"^phase/environment/(?P<slug>[\w-]+)/$",
views.get_challenge_phase_environment_url,
name="get_challenge_phase_environment_url",
),
]
71 changes: 65 additions & 6 deletions apps/challenges/utils.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
import os

import json
import logging
import uuid

from botocore.exceptions import ClientError
from moto import mock_ecr, mock_sts

from base.utils import get_model_object, get_boto3_client
from base.utils import (
get_model_object,
get_boto3_client,
mock_if_non_prod_aws,
)

from .models import (
Challenge,
Expand Down Expand Up @@ -81,9 +86,9 @@ def get_aws_credentials_for_challenge(challenge_pk):
}
else:
aws_keys = {
"AWS_ACCOUNT_ID": os.environ.get("AWS_ACCOUNT_ID"),
"AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID"),
"AWS_SECRET_ACCESS_KEY": os.environ.get("AWS_SECRET_ACCESS_KEY"),
"AWS_ACCOUNT_ID": os.environ.get("AWS_ACCOUNT_ID", "aws_account_id"),
"AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID", "aws_access_key_id"),
"AWS_SECRET_ACCESS_KEY": os.environ.get("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key"),
"AWS_REGION": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),
}
return aws_keys
Expand Down Expand Up @@ -119,7 +124,8 @@ def get_or_create_ecr_repository(name, aws_keys):
)
repository = response["repositories"][0]
except ClientError as e:
if e.response["Error"]["Code"] == "RepositoryNotFoundException":
if e.response["Error"]["Code"] == "RepositoryNotFoundException" or\
e.response["Error"]["Code"] == "400":
response = client.create_repository(repositoryName=name)
repository = response["repository"]
created = True
Expand Down Expand Up @@ -189,3 +195,56 @@ def create_federated_user(name, repository, aws_keys):
DurationSeconds=43200,
)
return response


@mock_if_non_prod_aws(mock_ecr)
@mock_if_non_prod_aws(mock_sts)
def get_aws_credentials_for_submission(challenge, participant_team):
"""
Method to generate AWS Credentails for CLI's Push
Wrappers:
- mock_ecr: To mock ECR requests to generate ecr credemntials
- mock_sts: To mock STS requests to generated federated user
Args:
- challenge: Challenge model
- participant_team: Participant Team Model
Returns:
- dict: {
"federated_user"
"docker_repository_uri"
}
"""
aws_keys = get_aws_credentials_for_challenge(challenge.pk)
ecr_repository_name = "{}-participant-team-{}".format(
challenge.slug, participant_team.pk
)
ecr_repository_name = convert_to_aws_ecr_compatible_format(
ecr_repository_name
)
repository, created = get_or_create_ecr_repository(
ecr_repository_name, aws_keys
)
name = str(uuid.uuid4())[:32]
docker_repository_uri = repository["repositoryUri"]
federated_user = create_federated_user(name, ecr_repository_name, aws_keys)
return {
"federated_user": federated_user,
"docker_repository_uri": docker_repository_uri,
}


def is_user_in_allowed_email_domains(email, challenge_pk):
challenge = get_challenge_model(challenge_pk)
for domain in challenge.allowed_email_domains:
if domain.lower() in email.lower():
return True
return False


def is_user_in_blocked_email_domains(email, challenge_pk):
challenge = get_challenge_model(challenge_pk)
for domain in challenge.blocked_email_domains:
domain = "@" + domain
if domain.lower() in email.lower():
return True
return False
Loading

0 comments on commit 96968d6

Please sign in to comment.