[WEB-2115] chore: implemented global paginator and handled project is…

…sues pagination v1 (#5432)

* chore: implemented global paginator and handled project issues paginated v1

* chore: updated order_by

* chore: updated updated_at parameter to updated_at__gte

* chore: changed updated_at__gte default value to None

apiserver/plane/app/urls/issue.py

@@ -20,6 +20,7 @@
     IssueViewSet,
     LabelViewSet,
     BulkArchiveIssuesEndpoint,
+    IssuePaginatedViewSet,
 )
 
 urlpatterns = [
@@ -38,6 +39,12 @@
         ),
         name="project-issue",
     ),
+    # updated v1 paginated issues
+    path(
+        "workspaces/<str:slug>/projects/<uuid:project_id>/v2/issues/",
+        IssuePaginatedViewSet.as_view({"get": "list"}),
+        name="project-issues-paginated",
+    ),
     path(
         "workspaces/<str:slug>/projects/<uuid:project_id>/issues/<uuid:pk>/",
         IssueViewSet.as_view(
@@ -303,5 +310,5 @@
             }
         ),
         name="project-issue-draft",
-    )
+    ),
 ]

apiserver/plane/app/views/__init__.py

@@ -112,6 +112,7 @@
     IssueViewSet,
     IssueUserDisplayPropertyEndpoint,
     BulkDeleteIssuesEndpoint,
+    IssuePaginatedViewSet,
 )
 
 from .issue.activity import (

apiserver/plane/app/views/issue/base.py

@@ -57,10 +57,10 @@
 from .. import BaseAPIView, BaseViewSet
 from plane.utils.user_timezone_converter import user_timezone_converter
 from plane.bgtasks.recent_visited_task import recent_visited_task
+from plane.utils.global_paginator import paginate
 
 
 class IssueListEndpoint(BaseAPIView):
-
     @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.VIEWER, ROLE.GUEST])
     def get(self, request, slug, project_id):
         issue_ids = request.GET.get("issues", False)
@@ -599,7 +599,6 @@ def destroy(self, request, slug, project_id, pk=None):
 
 
 class IssueUserDisplayPropertyEndpoint(BaseAPIView):
-
     @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST, ROLE.VIEWER])
     def patch(self, request, slug, project_id):
         issue_property = IssueUserProperty.objects.get(
@@ -630,10 +629,8 @@ def get(self, request, slug, project_id):
 
 
 class BulkDeleteIssuesEndpoint(BaseAPIView):
-
     @allow_permission([ROLE.ADMIN])
     def delete(self, request, slug, project_id):
-
         issue_ids = request.data.get("issue_ids", [])
 
         if not len(issue_ids):
@@ -654,3 +651,141 @@ def delete(self, request, slug, project_id):
             {"message": f"{total_issues} issues were deleted"},
             status=status.HTTP_200_OK,
         )
+
+
+class IssuePaginatedViewSet(BaseViewSet):
+    def get_queryset(self):
+        workspace_slug = self.kwargs.get("slug")
+        project_id = self.kwargs.get("project_id")
+
+        return (
+            Issue.issue_objects.filter(
+                workspace__slug=workspace_slug, project_id=project_id
+            )
+            .select_related("workspace", "project", "state", "parent")
+            .prefetch_related("assignees", "labels", "issue_module__module")
+            .annotate(cycle_id=F("issue_cycle__cycle_id"))
+            .annotate(
+                link_count=IssueLink.objects.filter(issue=OuterRef("id"))
+                .order_by()
+                .annotate(count=Func(F("id"), function="Count"))
+                .values("count")
+            )
+            .annotate(
+                attachment_count=IssueAttachment.objects.filter(
+                    issue=OuterRef("id")
+                )
+                .order_by()
+                .annotate(count=Func(F("id"), function="Count"))
+                .values("count")
+            )
+            .annotate(
+                sub_issues_count=Issue.issue_objects.filter(
+                    parent=OuterRef("id")
+                )
+                .order_by()
+                .annotate(count=Func(F("id"), function="Count"))
+                .values("count")
+            )
+        ).distinct()
+
+    def process_paginated_result(self, fields, results, timezone):
+        paginated_data = results.values(*fields)
+
+        # converting the datetime fields in paginated data
+        datetime_fields = ["created_at", "updated_at"]
+        paginated_data = user_timezone_converter(
+            paginated_data, datetime_fields, timezone
+        )
+
+        return paginated_data
+
+    @allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.VIEWER, ROLE.GUEST])
+    def list(self, request, slug, project_id):
+        cursor = request.GET.get("cursor", None)
+        is_description_required = request.GET.get("description", False)
+        updated_at = request.GET.get("updated_at__gte", None)
+
+        # required fields
+        required_fields = [
+            "id",
+            "name",
+            "state_id",
+            "sort_order",
+            "completed_at",
+            "estimate_point",
+            "priority",
+            "start_date",
+            "target_date",
+            "sequence_id",
+            "project_id",
+            "parent_id",
+            "cycle_id",
+            "created_at",
+            "updated_at",
+            "created_by",
+            "updated_by",
+            "is_draft",
+            "archived_at",
+            "deleted_at",
+            "module_ids",
+            "label_ids",
+            "assignee_ids",
+            "link_count",
+            "attachment_count",
+            "sub_issues_count",
+        ]
+
+        if is_description_required:
+            required_fields.append("description_html")
+
+        # querying issues
+        base_queryset = Issue.issue_objects.filter(
+            workspace__slug=slug, project_id=project_id
+        ).order_by("updated_at")
+        queryset = self.get_queryset().order_by("updated_at")
+
+        # filtering issues by greater then updated_at given by the user
+        if updated_at:
+            base_queryset = base_queryset.filter(updated_at__gte=updated_at)
+            queryset = queryset.filter(updated_at__gte=updated_at)
+
+        queryset = queryset.annotate(
+            label_ids=Coalesce(
+                ArrayAgg(
+                    "labels__id",
+                    distinct=True,
+                    filter=~Q(labels__id__isnull=True),
+                ),
+                Value([], output_field=ArrayField(UUIDField())),
+            ),
+            assignee_ids=Coalesce(
+                ArrayAgg(
+                    "assignees__id",
+                    distinct=True,
+                    filter=~Q(assignees__id__isnull=True)
+                    & Q(assignees__member_project__is_active=True),
+                ),
+                Value([], output_field=ArrayField(UUIDField())),
+            ),
+            module_ids=Coalesce(
+                ArrayAgg(
+                    "issue_module__module_id",
+                    distinct=True,
+                    filter=~Q(issue_module__module_id__isnull=True)
+                    & Q(issue_module__module__archived_at__isnull=True),
+                ),
+                Value([], output_field=ArrayField(UUIDField())),
+            ),
+        )
+
+        paginated_data = paginate(
+            base_queryset=base_queryset,
+            queryset=queryset,
+            cursor=cursor,
+            on_result=lambda results: self.process_paginated_result(
+                required_fields, results, request.user.user_timezone
+            ),
+        )
+
+        return Response(paginated_data, status=status.HTTP_200_OK)

apiserver/plane/utils/global_paginator.py

@@ -0,0 +1,78 @@
+# constants
+PAGINATOR_MAX_LIMIT = 1000
+
+
+class PaginateCursor:
+    def __init__(self, current_page_size: int, current_page: int, offset: int):
+        self.current_page_size = current_page_size
+        self.current_page = current_page
+        self.offset = offset
+
+    def __str__(self):
+        return f"{self.current_page_size}:{self.current_page}:{self.offset}"
+
+    @classmethod
+    def from_string(self, value):
+        """Return the cursor value from string format"""
+        try:
+            bits = value.split(":")
+            if len(bits) != 3:
+                raise ValueError(
+                    "Cursor must be in the format 'value:offset:is_prev'"
+                )
+            return self(int(bits[0]), int(bits[1]), int(bits[2]))
+        except (TypeError, ValueError) as e:
+            raise ValueError(f"Invalid cursor format: {e}")
+
+
+def paginate(base_queryset, queryset, cursor, on_result):
+    # validating for cursor
+    if cursor is None:
+        cursor_object = PaginateCursor(PAGINATOR_MAX_LIMIT, 0, 0)
+    else:
+        cursor_object = PaginateCursor.from_string(cursor)
+
+    # getting the issues count
+    total_results = base_queryset.count()
+    page_size = min(cursor_object.current_page_size, PAGINATOR_MAX_LIMIT)
+
+    # Calculate the start and end index for the paginated data
+    start_index = 0
+    if cursor_object.current_page > 0:
+        start_index = cursor_object.current_page * page_size
+    end_index = min(start_index + page_size, total_results)
+
+    # Get the paginated data
+    paginated_data = queryset[start_index:end_index]
+
+    # Create the pagination info object
+    prev_cursor = f"{page_size}:{cursor_object.current_page-1}:0"
+    cursor = f"{page_size}:{cursor_object.current_page}:0"
+    next_cursor = None
+    if end_index < total_results:
+        next_cursor = f"{page_size}:{cursor_object.current_page+1}:0"
+
+    prev_page_results = False
+    if cursor_object.current_page > 0:
+        prev_page_results = True
+
+    next_page_results = False
+    if next_cursor:
+        next_page_results = True
+
+    if on_result:
+        paginated_data = on_result(paginated_data)
+
+    # returning the result
+    paginated_data = {
+        "prev_cursor": prev_cursor,
+        "cursor": cursor,
+        "next_cursor": next_cursor,
+        "prev_page_results": prev_page_results,
+        "next_page_results": next_page_results,
+        "page_count": len(paginated_data),
+        "total_results": total_results,
+        "results": paginated_data,
+    }
+
+    return paginated_data