diff --git a/apps/base/utils.py b/apps/base/utils.py index 4a5d6aefd5..14bad1b0d1 100644 --- a/apps/base/utils.py +++ b/apps/base/utils.py @@ -154,7 +154,7 @@ def get_boto3_client(resource, aws_keys): logger.exception(e) -def get_sqs_queue_object(): +def get_or_create_sqs_queue_object(queue_name): if settings.DEBUG or settings.TEST: queue_name = "evalai_submission_queue" sqs = boto3.resource( @@ -212,19 +212,25 @@ def send_slack_notification(webhook=settings.SLACK_WEB_HOOK_URL, message=""): try: data = { "text": message["text"], - "attachments": [ - { - "color": "ffaf4b", - "fields": message["fields"] - } - ] + "attachments": [{"color": "ffaf4b", "fields": message["fields"]}], } return requests.post( webhook, data=json.dumps(data), - headers={"Content-Type": "application/json"} + headers={"Content-Type": "application/json"}, ) except Exception as e: logger.exception( - "Exception raised while sending slack notification. \n Exception message: {}".format(e) + "Exception raised while sending slack notification. \n Exception message: {}".format( + e + ) ) + + +def mock_if_non_prod_aws(aws_mocker): + def decorator(func): + if not (settings.DEBUG or settings.TEST): + return func + return aws_mocker(func) + + return decorator diff --git a/apps/challenges/admin.py b/apps/challenges/admin.py index 44eca234eb..c79ea49a08 100644 --- a/apps/challenges/admin.py +++ b/apps/challenges/admin.py @@ -1,5 +1,6 @@ from django import forms from django.contrib import admin, messages + from django.contrib.admin.helpers import ActionForm from base.admin import ImportExportTimeStampedAdmin @@ -12,6 +13,8 @@ stop_workers, ) +from .admin_filters import ChallengeFilter + from .models import ( Challenge, ChallengeConfiguration, @@ -52,6 +55,7 @@ class ChallengeAdmin(ImportExportTimeStampedAdmin): "task_def_arn", ) list_filter = ( + ChallengeFilter, "published", "is_registration_open", "enable_forum", diff --git a/apps/challenges/admin_filters.py b/apps/challenges/admin_filters.py new file mode 100644 index 0000000000..120ba37dd6 --- /dev/null +++ b/apps/challenges/admin_filters.py @@ -0,0 +1,38 @@ +from django.contrib.admin import SimpleListFilter +from django.utils import timezone + + +class ChallengeFilter(SimpleListFilter): + + title = "Challenges" + parameter_name = "challenge" + + def lookups(self, request, model_admin): + options = [ + ("past", "Past"), + ("present", "Ongoing"), + ("future", "Upcoming"), + ] + return options + + def queryset(self, request, queryset): + q_params = { + "published": True, + "approved_by_admin": True, + "is_disabled": False, + } + if self.value() == "past": + q_params["end_date__lt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges + + elif self.value() == "present": + q_params["start_date__lt"] = timezone.now() + q_params["end_date__gt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges + + elif self.value() == "future": + q_params["start_date__gt"] = timezone.now() + challenges = queryset.filter(**q_params) + return challenges diff --git a/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py b/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py new file mode 100644 index 0000000000..5b2872a00b --- /dev/null +++ b/apps/challenges/migrations/0058_adding_environment_url_for_rl_challenges.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-17 09:25 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('challenges', '0058_add_show_leaderboard_by_latest_submission_field_in_challenge_phase_split_model'), + ] + + operations = [ + migrations.AddField( + model_name='challengephase', + name='environment_url', + field=models.CharField(max_length=2128, null=True, validators=[django.core.validators.URLValidator()]), + ), + ] diff --git a/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py b/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py new file mode 100644 index 0000000000..2c111bf743 --- /dev/null +++ b/apps/challenges/migrations/0059_add_blank_in_phase_environment_url.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2019-12-24 17:28 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("challenges", "0058_adding_environment_url_for_rl_challenges") + ] + + operations = [ + migrations.AlterField( + model_name="challengephase", + name="environment_url", + field=models.CharField( + blank=True, + max_length=2128, + null=True, + validators=[django.core.validators.URLValidator()], + ), + ) + ] diff --git a/apps/challenges/models.py b/apps/challenges/models.py index cfa3a4e22a..ff975755c3 100644 --- a/apps/challenges/models.py +++ b/apps/challenges/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.contrib.auth.models import User +from django.core.validators import URLValidator from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils import timezone @@ -84,7 +85,7 @@ def __init__(self, *args, **kwargs): models.TextField(null=True, blank=True), default=[], blank=True, - null=True + null=True, ) remote_evaluation = models.BooleanField( default=False, verbose_name="Remote Evaluation", db_index=True @@ -122,11 +123,11 @@ def __init__(self, *args, **kwargs): max_length=20, verbose_name="evalai-cli version", null=True, blank=True ) # The number of active workers on Fargate of the challenge. - workers = models.IntegerField( - null=True, blank=True, default=None - ) + workers = models.IntegerField(null=True, blank=True, default=None) # The task definition ARN for the challenge, used for updating and creating service. - task_def_arn = models.CharField(null=True, blank=True, max_length=2048, default="") + task_def_arn = models.CharField( + null=True, blank=True, max_length=2048, default="" + ) class Meta: app_label = "challenges" @@ -170,7 +171,9 @@ def is_active(self): weak=False, ) signals.post_save.connect( - model_field_name(field_name="evaluation_script")(restart_workers_signal_callback), + model_field_name(field_name="evaluation_script")( + restart_workers_signal_callback + ), sender=Challenge, weak=False, ) @@ -232,6 +235,9 @@ def __init__(self, *args, **kwargs): null=True, ) slug = models.SlugField(max_length=200, null=True, unique=True) + environment_url = models.CharField( + validators=[URLValidator()], null=True, blank=True, max_length=2128 + ) # Max length of URL and tag is 2000 and 128 respectively class Meta: app_label = "challenges" @@ -280,7 +286,9 @@ def save(self, *args, **kwargs): weak=False, ) signals.post_save.connect( - model_field_name(field_name="test_annotation")(restart_workers_signal_callback), + model_field_name(field_name="test_annotation")( + restart_workers_signal_callback + ), sender=ChallengePhase, weak=False, ) diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py index c2a88bd438..ac14d8512b 100644 --- a/apps/challenges/serializers.py +++ b/apps/challenges/serializers.py @@ -113,6 +113,7 @@ class Meta: "challenge_phase_name", "dataset_split_name", "visibility", + "show_leaderboard_by_latest_submission" ) def get_dataset_split_name(self, obj): diff --git a/apps/challenges/urls.py b/apps/challenges/urls.py index 61d9865bc1..3f44c5b8dd 100644 --- a/apps/challenges/urls.py +++ b/apps/challenges/urls.py @@ -151,4 +151,9 @@ views.get_challenge_phase_by_slug, name="get_challenge_phase_by_slug", ), + url( + r"^phase/environment/(?P[\w-]+)/$", + views.get_challenge_phase_environment_url, + name="get_challenge_phase_environment_url", + ), ] diff --git a/apps/challenges/utils.py b/apps/challenges/utils.py index 3294deb0fc..dfdab5491c 100644 --- a/apps/challenges/utils.py +++ b/apps/challenges/utils.py @@ -1,11 +1,16 @@ import os - import json import logging +import uuid from botocore.exceptions import ClientError +from moto import mock_ecr, mock_sts -from base.utils import get_model_object, get_boto3_client +from base.utils import ( + get_model_object, + get_boto3_client, + mock_if_non_prod_aws, +) from .models import ( Challenge, @@ -81,9 +86,9 @@ def get_aws_credentials_for_challenge(challenge_pk): } else: aws_keys = { - "AWS_ACCOUNT_ID": os.environ.get("AWS_ACCOUNT_ID"), - "AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID"), - "AWS_SECRET_ACCESS_KEY": os.environ.get("AWS_SECRET_ACCESS_KEY"), + "AWS_ACCOUNT_ID": os.environ.get("AWS_ACCOUNT_ID", "aws_account_id"), + "AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID", "aws_access_key_id"), + "AWS_SECRET_ACCESS_KEY": os.environ.get("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key"), "AWS_REGION": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), } return aws_keys @@ -119,7 +124,8 @@ def get_or_create_ecr_repository(name, aws_keys): ) repository = response["repositories"][0] except ClientError as e: - if e.response["Error"]["Code"] == "RepositoryNotFoundException": + if e.response["Error"]["Code"] == "RepositoryNotFoundException" or\ + e.response["Error"]["Code"] == "400": response = client.create_repository(repositoryName=name) repository = response["repository"] created = True @@ -189,3 +195,56 @@ def create_federated_user(name, repository, aws_keys): DurationSeconds=43200, ) return response + + +@mock_if_non_prod_aws(mock_ecr) +@mock_if_non_prod_aws(mock_sts) +def get_aws_credentials_for_submission(challenge, participant_team): + """ + Method to generate AWS Credentails for CLI's Push + Wrappers: + - mock_ecr: To mock ECR requests to generate ecr credemntials + - mock_sts: To mock STS requests to generated federated user + Args: + - challenge: Challenge model + - participant_team: Participant Team Model + Returns: + - dict: { + "federated_user" + "docker_repository_uri" + } + """ + aws_keys = get_aws_credentials_for_challenge(challenge.pk) + ecr_repository_name = "{}-participant-team-{}".format( + challenge.slug, participant_team.pk + ) + ecr_repository_name = convert_to_aws_ecr_compatible_format( + ecr_repository_name + ) + repository, created = get_or_create_ecr_repository( + ecr_repository_name, aws_keys + ) + name = str(uuid.uuid4())[:32] + docker_repository_uri = repository["repositoryUri"] + federated_user = create_federated_user(name, ecr_repository_name, aws_keys) + return { + "federated_user": federated_user, + "docker_repository_uri": docker_repository_uri, + } + + +def is_user_in_allowed_email_domains(email, challenge_pk): + challenge = get_challenge_model(challenge_pk) + for domain in challenge.allowed_email_domains: + if domain.lower() in email.lower(): + return True + return False + + +def is_user_in_blocked_email_domains(email, challenge_pk): + challenge = get_challenge_model(challenge_pk) + for domain in challenge.blocked_email_domains: + domain = "@" + domain + if domain.lower() in email.lower(): + return True + return False diff --git a/apps/challenges/views.py b/apps/challenges/views.py index 4c3b4bfea3..c75d064d47 100644 --- a/apps/challenges/views.py +++ b/apps/challenges/views.py @@ -52,6 +52,8 @@ get_challenge_phase_split_model, get_dataset_split_model, get_leaderboard_model, + is_user_in_allowed_email_domains, + is_user_in_blocked_email_domains ) from hosts.models import ChallengeHost, ChallengeHostTeam from hosts.utils import ( @@ -96,11 +98,8 @@ ZipChallengePhaseSplitSerializer, ) from .utils import ( - create_federated_user, - convert_to_aws_ecr_compatible_format, - get_aws_credentials_for_challenge, get_file_content, - get_or_create_ecr_repository, + get_aws_credentials_for_submission, ) logger = logging.getLogger(__name__) @@ -273,12 +272,7 @@ def add_participant_team_to_challenge( # Check if user is in allowed list. user_email = request.user.email if len(challenge.allowed_email_domains) > 0: - present = False - for domain in challenge.allowed_email_domains: - if domain.lower() in user_email.lower(): - present = True - break - if not present: + if not is_user_in_allowed_email_domains(user_email, challenge_pk): message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge." domains = "" for domain in challenge.allowed_email_domains: @@ -290,18 +284,16 @@ def add_participant_team_to_challenge( ) # Check if user is in blocked list. - for domain in challenge.blocked_email_domains: - domain = "@" + domain - if domain.lower() in user_email.lower(): - message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." - domains = "" - for domain in challenge.blocked_email_domains: - domains = "{}{}{}".format(domains, "/", domain) - domains = domains[1:] - response_data = {"error": message.format(domains)} - return Response( - response_data, status=status.HTTP_406_NOT_ACCEPTABLE - ) + if is_user_in_blocked_email_domains(user_email, challenge_pk): + message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." + domains = "" + for domain in challenge.blocked_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) # check to disallow the user if he is a Challenge Host for this challenge participant_team_user_ids = set( @@ -1901,8 +1893,19 @@ def get_broker_url_by_challenge_pk(request, challenge_pk): @authentication_classes((ExpiringTokenAuthentication,)) def get_aws_credentials_for_participant_team(request, phase_pk): """ - Returns: - Dictionary containing AWS credentials for the participant team for a particular challenge + Endpoint to generate AWS Credentails for CLI + Args: + - challenge: Challenge model + - participant_team: Participant Team Model + Returns: + - JSON: { + "federated_user" + "docker_repository_uri" + } + Raises: + - BadRequestException 400 + - When participant_team has not participanted in challenge + - When Challenge is not Docker based """ challenge_phase = get_challenge_phase_model(phase_pk) challenge = challenge_phase.challenge @@ -1920,24 +1923,7 @@ def get_aws_credentials_for_participant_team(request, phase_pk): "error": "You have not participated in this challenge." } return Response(response_data, status=status.HTTP_400_BAD_REQUEST) - - aws_keys = get_aws_credentials_for_challenge(challenge.pk) - ecr_repository_name = "{}-participant-team-{}".format( - challenge.slug, participant_team.pk - ) - ecr_repository_name = convert_to_aws_ecr_compatible_format( - ecr_repository_name - ) - repository, created = get_or_create_ecr_repository( - ecr_repository_name, aws_keys - ) - name = str(uuid.uuid4())[:32] - docker_repository_uri = repository["repositoryUri"] - federated_user = create_federated_user(name, ecr_repository_name, aws_keys) - data = { - "federated_user": federated_user, - "docker_repository_uri": docker_repository_uri, - } + data = get_aws_credentials_for_submission(challenge, participant_team) response_data = {"success": data} return Response(response_data, status=status.HTTP_200_OK) @@ -2176,3 +2162,35 @@ def get_challenge_phase_by_slug(request, slug): serializer = ChallengePhaseSerializer(challenge_phase) response_data = serializer.data return Response(response_data, status=status.HTTP_200_OK) + + +@api_view(["GET"]) +@throttle_classes([UserRateThrottle]) +@permission_classes((permissions.IsAuthenticated, HasVerifiedEmail)) +@authentication_classes((ExpiringTokenAuthentication,)) +def get_challenge_phase_environment_url(request, slug): + """ + Returns environment image url and tag required for RL challenge evaluation + """ + try: + challenge_phase = ChallengePhase.objects.get(slug=slug) + except ChallengePhase.DoesNotExist: + response_data = { + "error": "Challenge phase with slug {} does not exist".format(slug) + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + challenge = get_challenge_model(challenge_phase.challenge.pk) + if not is_user_a_host_of_challenge(request.user, challenge.pk): + response_data = { + "error": "Sorry, you are not authorized to access test environment URL." + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + if not challenge.is_docker_based: + response_data = { + "error": "The challenge doesn't require uploading Docker images, hence no test environment URL." + } + return Response(response_data, status=status.HTTP_400_BAD_REQUEST) + response_data = { + "environment_url": challenge_phase.environment_url + } + return Response(response_data, status=status.HTTP_200_OK) diff --git a/apps/hosts/models.py b/apps/hosts/models.py index b58af8da54..d5fa3bd756 100644 --- a/apps/hosts/models.py +++ b/apps/hosts/models.py @@ -10,7 +10,7 @@ class ChallengeHostTeam(TimeStampedModel): """ - Model representing the Host Team for a partiuclar challenge + Model representing the Host Team for a particular challenge """ team_name = models.CharField(max_length=100, unique=True) diff --git a/apps/jobs/admin.py b/apps/jobs/admin.py index 3a8e046d31..34e250779c 100644 --- a/apps/jobs/admin.py +++ b/apps/jobs/admin.py @@ -1,13 +1,13 @@ import logging +import requests -from django.contrib import admin +from django.contrib import admin, messages from base.admin import ImportExportTimeStampedAdmin from .models import Submission from .sender import publish_submission_message - logger = logging.getLogger(__name__) @@ -34,6 +34,7 @@ class SubmissionAdmin(ImportExportTimeStampedAdmin): "stderr_file", "submission_result_file", "submission_metadata_file", + "job_name", ) list_filter = ( "challenge_phase__challenge", @@ -69,9 +70,30 @@ def submit_job_to_worker(self, request, queryset): challenge_id, challenge_phase_id, submission_id ) ) - publish_submission_message( - challenge_id, challenge_phase_id, submission.id - ) + message = { + "challenge_pk": challenge_id, + "phase_pk": challenge_phase_id, + "submission_pk": submission.id, + } + + if submission.challenge_phase.challenge.is_docker_based: + try: + response = requests.get(submission.input_file.url) + except Exception as e: + messages.error( + request, + "Failed to get input_file with exception: {0}".format( + e + ), + ) + return + + if response and response.status_code == 200: + message["submitted_image_uri"] = response.json()[ + "submitted_image_uri" + ] + + publish_submission_message(message) queryset.update(status=Submission.SUBMITTED) submit_job_to_worker.short_description = "Run selected submissions" diff --git a/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py b/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py new file mode 100644 index 0000000000..eac39a7d20 --- /dev/null +++ b/apps/jobs/migrations/0013_add_job_id_field_in_submission_model.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-01-20 04:55 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("jobs", "0012_add_baseline_submission")] + + operations = [ + migrations.AddField( + model_name="submission", + name="job_id", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(blank=True, null=True), + blank=True, + default=[], + null=True, + size=None, + ), + ) + ] diff --git a/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py b/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py new file mode 100644 index 0000000000..159e4d49c2 --- /dev/null +++ b/apps/jobs/migrations/0014_rename_job_id_field_in_submission_model_to_job_name.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-01-20 05:23 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("jobs", "0013_add_job_id_field_in_submission_model")] + + operations = [ + migrations.RenameField( + model_name="submission", old_name="job_id", new_name="job_name" + ) + ] diff --git a/apps/jobs/models.py b/apps/jobs/models.py index 7da628f6ad..ba53431466 100644 --- a/apps/jobs/models.py +++ b/apps/jobs/models.py @@ -3,6 +3,7 @@ import logging from django.contrib.auth.models import User +from django.contrib.postgres.fields import ArrayField from django.db import models from django.db.models import Max from rest_framework.exceptions import PermissionDenied @@ -107,6 +108,12 @@ class Submission(TimeStampedModel): publication_url = models.CharField(max_length=1000, default="", blank=True) project_url = models.CharField(max_length=1000, default="", blank=True) is_baseline = models.BooleanField(default=False) + job_name = ArrayField( + models.TextField(null=True, blank=True), + default=[], + blank=True, + null=True, + ) def __str__(self): return "{}".format(self.id) diff --git a/apps/jobs/sender.py b/apps/jobs/sender.py index 2a1aaf12b1..0747b4a4f7 100644 --- a/apps/jobs/sender.py +++ b/apps/jobs/sender.py @@ -55,27 +55,24 @@ def get_or_create_sqs_queue(queue_name): return queue -def publish_submission_message(challenge_pk, phase_pk, submission_pk): +def publish_submission_message(message): """ Args: - challenge_pk: Challenge Id - phase_pk: Challenge Phase Id - submission_pk: Submission Id + message: A Dict with following keys + - "challenge_pk": int + - "phase_pk": int + - "submission_pk": int + - "submitted_image_uri": str, (only available when the challenge is a code upload challenge) Returns: Returns SQS response """ - message = { - "challenge_pk": challenge_pk, - "phase_pk": phase_pk, - "submission_pk": submission_pk, - } try: - challenge = Challenge.objects.get(pk=challenge_pk) + challenge = Challenge.objects.get(pk=message["challenge_pk"]) except Challenge.DoesNotExist: logger.exception( - "Challenge does not exist for the given id {}".format(challenge_pk) + "Challenge does not exist for the given id {}".format(message["challenge_pk"]) ) return queue_name = challenge.queue diff --git a/apps/jobs/serializers.py b/apps/jobs/serializers.py index ab848a3950..c6deaea9eb 100644 --- a/apps/jobs/serializers.py +++ b/apps/jobs/serializers.py @@ -50,6 +50,7 @@ class Meta: "submission_result_file", "when_made_public", "is_baseline", + "job_name", ) def get_participant_team_name(self, obj): diff --git a/apps/jobs/tasks.py b/apps/jobs/tasks.py index f57fd4d23f..6f3db2ea93 100644 --- a/apps/jobs/tasks.py +++ b/apps/jobs/tasks.py @@ -73,7 +73,11 @@ def download_file_and_publish_submission_message( submission = serializer.instance # publish messages in the submission worker queue - publish_submission_message(challenge_phase.challenge.pk, challenge_phase.pk, submission.pk) + publish_submission_message({ + "challenge_pk": challenge_phase.challenge.pk, + "phase_pk": challenge_phase.pk, + "submission_pk": submission.pk + }) logger.info("Message published to submission worker successfully!") shutil.rmtree(downloaded_file['temp_dir_path']) except Exception as e: diff --git a/apps/jobs/urls.py b/apps/jobs/urls.py index 42f6dac14a..82418c0a2c 100644 --- a/apps/jobs/urls.py +++ b/apps/jobs/urls.py @@ -46,7 +46,7 @@ name="get_submissions_for_challenge", ), url( - r"^queues/(?P[\w-]+)/receipt/(?P[\w-]+)/$", + r"^queues/(?P[\w-]+)/$", views.delete_submission_message_from_queue, name="delete_submission_message_from_queue", ), diff --git a/apps/jobs/views.py b/apps/jobs/views.py index cc91fcd473..db88387aaf 100644 --- a/apps/jobs/views.py +++ b/apps/jobs/views.py @@ -3,6 +3,7 @@ import json import logging +import requests from rest_framework import permissions, status from rest_framework.decorators import ( api_view, @@ -29,7 +30,7 @@ from base.utils import ( paginated_queryset, StandardResultSetPagination, - get_sqs_queue_object, + get_or_create_sqs_queue_object, get_boto3_client, ) from challenges.models import ( @@ -276,14 +277,32 @@ def challenge_submission(request, challenge_id, challenge_phase_id): "request": request, }, ) + message = { + "challenge_pk": challenge_id, + "phase_pk": challenge_phase_id, + } + if challenge.is_docker_based: + try: + file_content = json.loads(request.FILES["input_file"].read()) + message["submitted_image_uri"] = file_content[ + "submitted_image_uri" + ] + except Exception as ex: + response_data = { + "error": "Error {} in submitted_image_uri from submission file".format( + ex + ) + } + return Response( + response_data, status=status.HTTP_400_BAD_REQUEST + ) if serializer.is_valid(): serializer.save() response_data = serializer.data submission = serializer.instance + message["submission_pk"] = submission.id # publish message in the queue - publish_submission_message( - challenge_id, challenge_phase_id, submission.id - ) + publish_submission_message(message) return Response(response_data, status=status.HTTP_201_CREATED) return Response( serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE @@ -980,11 +999,20 @@ def update_submission(request, challenge_pk): if request.method == "PATCH": submission_pk = request.data.get("submission") submission_status = request.data.get("submission_status", "").lower() + job_name = request.data.get("job_name", "").lower() submission = get_submission_model(submission_pk) + jobs = submission.job_name + if job_name: + jobs.append(job_name) if submission_status not in [Submission.RUNNING]: response_data = {"error": "Sorry, submission status is invalid"} return Response(response_data, status=status.HTTP_400_BAD_REQUEST) - data = {"status": submission_status, "started_at": timezone.now()} + + data = { + "status": submission_status, + "started_at": timezone.now(), + "job_name": jobs, + } serializer = SubmissionSerializer( submission, data=data, partial=True, context={"request": request} ) @@ -1027,7 +1055,31 @@ def re_run_submission(request, submission_pk): } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) - publish_submission_message(challenge.pk, challenge_phase.pk, submission.pk) + message = { + "challenge_pk": challenge.pk, + "phase_pk": challenge_phase.pk, + "submission_pk": submission.pk, + } + + if submission.challenge_phase.challenge.is_docker_based: + try: + response = requests.get(submission.input_file) + except Exception as e: + response_data = { + "error": "Failed to get submission input file with error: {0}".format( + e + ) + } + return Response( + response_data, status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + if response and response.status_code == 200: + message["submitted_image_uri"] = response.json()[ + "submitted_image_uri" + ] + + publish_submission_message(message) response_data = { "success": "Submission is successfully submitted for re-running" } @@ -1102,7 +1154,7 @@ def get_submission_message_from_queue(request, queue_name): } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) - queue = get_sqs_queue_object() + queue = get_or_create_sqs_queue_object(queue_name) try: messages = queue.receive_messages() if len(messages): @@ -1129,11 +1181,11 @@ def get_submission_message_from_queue(request, queue_name): return Response(response_data, status=status.HTTP_400_BAD_REQUEST) -@api_view(["GET"]) +@api_view(["POST"]) @throttle_classes([UserRateThrottle]) @permission_classes((permissions.IsAuthenticated, HasVerifiedEmail)) @authentication_classes((ExpiringTokenAuthentication,)) -def delete_submission_message_from_queue(request, queue_name, receipt_handle): +def delete_submission_message_from_queue(request, queue_name): """ API to delete submission message from AWS SQS queue Arguments: @@ -1151,13 +1203,14 @@ def delete_submission_message_from_queue(request, queue_name, receipt_handle): return Response(response_data, status=status.HTTP_400_BAD_REQUEST) challenge_pk = challenge.pk + receipt_handle = request.data["receipt_handle"] if not is_user_a_host_of_challenge(request.user, challenge_pk): response_data = { "error": "Sorry, you are not authorized to access this resource" } return Response(response_data, status=status.HTTP_401_UNAUTHORIZED) - queue = get_sqs_queue_object() + queue = get_or_create_sqs_queue_object(queue_name) try: message = queue.Message(receipt_handle) message.delete() diff --git a/apps/participants/admin.py b/apps/participants/admin.py index afc7efb606..8cf646596f 100644 --- a/apps/participants/admin.py +++ b/apps/participants/admin.py @@ -22,7 +22,7 @@ class ParticipantAdmin(ImportExportTimeStampedAdmin): list_display = ("user", "status", "team") search_fields = ("user__username", "status", "team__team_name") - list_filter = ("status", "team") + list_filter = ("status",) resource_class = ParticipantResource @@ -34,4 +34,4 @@ class ParticipantTeamAdmin(ImportExportTimeStampedAdmin): """ list_display = ("team_name", "get_all_participants_email", "team_url") - list_filter = ("team_name",) + search_fields = ("team_name", "team_url", "created_by__username") diff --git a/apps/participants/views.py b/apps/participants/views.py index 9629490563..43a2e16a58 100644 --- a/apps/participants/views.py +++ b/apps/participants/views.py @@ -17,6 +17,11 @@ from base.utils import paginated_queryset from challenges.models import Challenge from challenges.serializers import ChallengeSerializer +from challenges.utils import ( + get_challenge_model, + is_user_in_allowed_email_domains, + is_user_in_blocked_email_domains +) from hosts.utils import is_user_a_host_of_challenge from .models import Participant, ParticipantTeam @@ -197,6 +202,55 @@ def invite_participant_to_team(request, pk): } return Response(response_data, status=status.HTTP_406_NOT_ACCEPTABLE) + if len(team_participated_challenges) > 0: + for challenge_pk in team_participated_challenges: + challenge = get_challenge_model(challenge_pk) + + if len(challenge.banned_email_ids) > 0: + # Check if team participants emails are banned + for participant_email in participant_team.get_all_participants_email(): + if participant_email in challenge.banned_email_ids: + message = "You cannot invite as you're a part of {} team and it has been banned " + "from this challenge. Please contact the challenge host." + response_data = {"error": message.format(participant_team.team_name)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if invited user is banned + if email in challenge.banned_email_ids: + message = "You cannot invite as the invited user has been banned " + "from this challenge. Please contact the challenge host." + response_data = {"error": message} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if user is in allowed list. + if len(challenge.allowed_email_domains) > 0: + if not is_user_in_allowed_email_domains(email, challenge_pk): + message = "Sorry, users with {} email domain(s) are only allowed to participate in this challenge." + domains = "" + for domain in challenge.allowed_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + + # Check if user is in blocked list. + if is_user_in_blocked_email_domains(email, challenge_pk): + message = "Sorry, users with {} email domain(s) are not allowed to participate in this challenge." + domains = "" + for domain in challenge.blocked_email_domains: + domains = "{}{}{}".format(domains, "/", domain) + domains = domains[1:] + response_data = {"error": message.format(domains)} + return Response( + response_data, status=status.HTTP_406_NOT_ACCEPTABLE + ) + serializer = InviteParticipantToTeamSerializer( data=request.data, context={"participant_team": participant_team, "request": request}, diff --git a/docs/source/architecture.md b/docs/source/architecture.md index d3b14df209..abfda434de 100644 --- a/docs/source/architecture.md +++ b/docs/source/architecture.md @@ -6,7 +6,7 @@ EvalAI helps researchers, students, and data scientists to create, collaborate, #### Django -Django is the heart of the application, which powers our backend. We use Django version 1.11.18. +Django is the heart of the application, which powers our backend. We use Django version 1.11.23. #### Django Rest Framework diff --git a/docs/source/faq(developers).md b/docs/source/faq(developers).md index 0c3ef62934..e3d006a4ce 100644 --- a/docs/source/faq(developers).md +++ b/docs/source/faq(developers).md @@ -9,7 +9,7 @@ Alternatively, if you come across a new bug on the site, please file a new issue Please refer to [Technologies Used](https://evalai.readthedocs.io/en/latest/architecture.html) -#### Q. Where could I learn Github Commands? +#### Q. Where could I learn GitHub Commands? Refer to [GitHub Guide](https://help.github.com/articles/git-and-github-learning-resources/). diff --git a/frontend/src/js/controllers/challengeCtrl.js b/frontend/src/js/controllers/challengeCtrl.js index f7df68bb81..3b19106b8d 100644 --- a/frontend/src/js/controllers/challengeCtrl.js +++ b/frontend/src/js/controllers/challengeCtrl.js @@ -1891,10 +1891,9 @@ vm.challengePhaseDialog = function(ev, phase) { vm.page.challenge_phase = phase; vm.page.max_submissions_per_day = phase.max_submissions_per_day; - vm.phaseStartDate = phase.start_date; - vm.phaseStartDate = moment(vm.phaseStartDate); - vm.phaseEndDate = phase.end_date; - vm.phaseEndDate = moment(vm.phaseEndDate); + vm.page.max_submissions_per_month = phase.max_submissions_per_month; + vm.phaseStartDate = moment(phase.start_date); + vm.phaseEndDate = moment(phase.end_date); vm.testAnnotationFile = null; vm.sanityCheckPass = true; vm.sanityCheck = ""; @@ -1918,6 +1917,7 @@ formData.append("start_date", vm.phaseStartDate.toISOString()); formData.append("end_date", vm.phaseEndDate.toISOString()); formData.append("max_submissions_per_day", vm.page.challenge_phase.max_submissions_per_day); + formData.append("max_submissions_per_month", vm.page.challenge_phase.max_submissions_per_month); formData.append("max_submissions", vm.page.challenge_phase.max_submissions); if (vm.testAnnotationFile) { formData.append("test_annotation", vm.testAnnotationFile); @@ -2008,6 +2008,63 @@ }); }; + // Edit Challenge Start and End Date + vm.challengeDateDialog = function(ev) { + vm.challengeStartDate = moment(vm.page.start_date); + vm.challengeEndDate = moment(vm.page.end_date); + $mdDialog.show({ + scope: $scope, + preserveScope: true, + targetEvent: ev, + templateUrl: 'dist/views/web/challenge/edit-challenge/edit-challenge-date.html', + escapeToClose: false + }); + }; + + vm.editChallengeDate = function(editChallengeDateForm) { + if (editChallengeDateForm) { + var challengeHostList = utilities.getData("challengeCreator"); + for (var challenge in challengeHostList) { + if (challenge == vm.challengeId) { + vm.challengeHostId = challengeHostList[challenge]; + break; + } + } + parameters.url = "challenges/challenge_host_team/" + vm.challengeHostId + "/challenge/" + vm.challengeId; + parameters.method = 'PATCH'; + if (new Date(vm.challengeStartDate).valueOf() < new Date(vm.challengeEndDate).valueOf()) { + parameters.data = { + "start_date": vm.challengeStartDate, + "end_date": vm.challengeEndDate + }; + parameters.callback = { + onSuccess: function(response) { + var status = response.status; + utilities.hideLoader(); + if (status === 200) { + vm.page.start_date = vm.challengeStartDate.format("MMM D, YYYY h:mm:ss A"); + vm.page.end_date = vm.challengeEndDate.format("MMM D, YYYY h:mm:ss A"); + $mdDialog.hide(); + $rootScope.notify("success", "The challenge start and end date is successfully updated!"); + } + }, + onError: function(response) { + utilities.hideLoader(); + $mdDialog.hide(); + var error = response.data; + $rootScope.notify("error", error); + } + }; + utilities.showLoader(); + utilities.sendRequest(parameters); + } else { + $rootScope.notify("error", "The challenge start date cannot be same or greater than end date."); + } + } else { + $mdDialog.hide(); + } + }; + $scope.$on('$destroy', function() { vm.stopFetchingSubmissions(); vm.stopLeaderboard(); diff --git a/frontend/src/views/web/challenge/challenge-page.html b/frontend/src/views/web/challenge/challenge-page.html index 24c292cb20..8e6d025e92 100644 --- a/frontend/src/views/web/challenge/challenge-page.html +++ b/frontend/src/views/web/challenge/challenge-page.html @@ -42,6 +42,28 @@ +
+ + Starts on: + {{ challenge.page.start_date | date:'medium' }} + +   + + + + + +
+ + Ends on: + {{ challenge.page.end_date | date:'medium' }} + +   + + + + +

diff --git a/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html b/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html new file mode 100644 index 0000000000..57c1a35bb6 --- /dev/null +++ b/frontend/src/views/web/challenge/edit-challenge/edit-challenge-date.html @@ -0,0 +1,43 @@ +
+
+
+
+
Edit Challenge Start and End Date
+
+ Start date and time + + +
+
+ End date and time + + +
+
+
    +
  • + Cancel +
    • +
  • + +
    • +
+
+
+
+
+
diff --git a/frontend/src/views/web/get-involved.html b/frontend/src/views/web/get-involved.html index 2748dabe8f..c3d4c288c7 100644 --- a/frontend/src/views/web/get-involved.html +++ b/frontend/src/views/web/get-involved.html @@ -16,7 +16,7 @@
Report issues
EvalAI Google Group, or contact us at team@cloudcv.org.

Improving and maintaining the site
-

The EvalAI project is fully open source, and is maintained by a large community of volunteers on Github. +

The EvalAI project is fully open source, and is maintained by a large community of volunteers on GitHub. We are in need of coders and designers so if you would like to help out, please drop us a line! The best way to get started is to write us at team@cloudcv.org or ping us on our Gitter Channel.

diff --git a/frontend/src/views/web/landing.html b/frontend/src/views/web/landing.html index 3e9d1a26e6..fd44091f9b 100644 --- a/frontend/src/views/web/landing.html +++ b/frontend/src/views/web/landing.html @@ -8,7 +8,7 @@

Evaluating state-of-the-art in AI

EvalAI is an open source platform for evaluating and - comparing machine learning (ML) and artificial intelligence algorithms (AI) at scale.

+ comparing machine learning (ML) and artificial intelligence (AI) algorithms at scale.

  • @@ -26,6 +26,76 @@

    Evaluating state-of-the-art in AI

+
+
+
+
+
+
+

50+

+
+
+
+
+
+ Hosted AI Challenges +
+
+
+
+
+
+
+
+
+

5,500+

+
+
+
+
+
+ Users +
+
+
+
+
+
+
+
+
+

51,000+

+
+
+
+
+
+ Submissions +
+
+
+
+
+
+
+
+
+

20+

+
+
+
+
+
+ Organizations +
+
+
+
+
+
+
+
+

Features

@@ -97,7 +167,7 @@
Faster evaluation
-
+

Popular challenges

@@ -139,7 +209,7 @@

Popular challenges

-
+
@@ -260,7 +330,7 @@

Partner Organizations

@@ -269,7 +339,7 @@

Partner Organizations

-
+

Cite our work

diff --git a/frontend/src/views/web/profile.html b/frontend/src/views/web/profile.html index e810b8fb9f..9859d223cd 100644 --- a/frontend/src/views/web/profile.html +++ b/frontend/src/views/web/profile.html @@ -43,7 +43,7 @@