[DRAFT] feat: simplify telemetry

.github/workflows/argilla-frontend.deploy-environment.yml

@@ -63,7 +63,7 @@ jobs:
             ADMIN_API_KEY=${{ steps.credentials.outputs.admin }}
             ANNOTATOR_PASSWORD=${{ steps.credentials.outputs.annotator }}
             ANNOTATOR_API_KEY=${{ steps.credentials.outputs.annotator }}
-            ARGILLA_ENABLE_TELEMETRY=0
+            HF_HUB_DISABLE_TELEMETRY=1
             API_BASE_URL=https://dev.argilla.io/
 
       - name: Post credentials in Slack

.github/workflows/argilla-server.yml

@@ -52,7 +52,7 @@ jobs:
           - 5432:5432
 
     env:
-      ARGILLA_ENABLE_TELEMETRY: 0
+      HF_HUB_DISABLE_TELEMETRY: 1
 
     steps:
       - name: Checkout Code 🛎

argilla-server/pyproject.toml

@@ -57,6 +57,8 @@ dependencies = [
     "typer >= 0.6.0, < 0.10.0", # spaCy only supports typer<0.10.0
     "packaging>=23.2",
     "psycopg2-binary>=2.9.9",
+    # For Telemetry
+    "huggingface_hub>=0.13,<1",
 ]
 
 [project.optional-dependencies]
@@ -106,7 +108,7 @@ log_format = "%(asctime)s %(name)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_cli = "True"
 testpaths = ["tests"]
-env = ["ARGILLA_ENABLE_TELEMETRY=0"]
+env = ["HF_HUB_DISABLE_TELEMETRY=0"]
 
 [tool.coverage.run]
 concurrency = ["greenlet", "thread", "multiprocessing"]

argilla-server/src/argilla_server/_app.py

@@ -40,6 +40,7 @@
 from argilla_server.search_engine import get_search_engine
 from argilla_server.settings import settings
 from argilla_server.static_rewrite import RewriteStaticFiles
+from argilla_server.telemetry import get_telemetry_client
 
 _LOGGER = logging.getLogger("argilla")
 
@@ -50,7 +51,9 @@
     show_telemetry_warning()
     await configure_database()
     await configure_search_engine()
+    await get_telemetry_client().track_server_startup()
     yield
+    await get_telemetry_client().track_server_shutdown()
 
 
 def create_server_app() -> FastAPI:
@@ -174,10 +177,14 @@
             "    https://docs.argilla.io/latest/reference/argilla-server/telemetry/\n\n"
             "Telemetry is currently enabled. If you want to disable it, you can configure\n"
             "the environment variable before relaunching the server:\n\n"
-            f'{"#set ARGILLA_ENABLE_TELEMETRY=0" if os.name == "nt" else "$>export ARGILLA_ENABLE_TELEMETRY=0"}'
+            f'{"#set HF_HUB_DISABLE_TELEMETRY=1" if os.name == "nt" else "$>export HF_HUB_DISABLE_TELEMETRY=1"}'
         )
         _LOGGER.warning(message)
 
+    message += "\n\n    "
+    message += "#set HF_HUB_DISABLE_TELEMETRY=1" if os.name == "nt" else "$>export HF_HUB_DISABLE_TELEMETRY=1"
+    message += "\n"
+
 
 async def _create_oauth_allowed_workspaces(db: AsyncSession):
     from argilla_server.security.settings import settings as security_settings

argilla-server/src/argilla_server/api/handlers/v1/datasets/datasets.py

@@ -22,14 +22,14 @@
 from argilla_server.api.policies.v1 import DatasetPolicy, MetadataPropertyPolicy, authorize, is_authorized
 from argilla_server.api.schemas.v1.datasets import (
     Dataset as DatasetSchema,
-    UsersProgress,
 )
 from argilla_server.api.schemas.v1.datasets import (
     DatasetCreate,
     DatasetMetrics,
     DatasetProgress,
     Datasets,
     DatasetUpdate,
+    UsersProgress,
 )
 from argilla_server.api.schemas.v1.fields import Field, FieldCreate, Fields
 from argilla_server.api.schemas.v1.metadata_properties import (
@@ -73,12 +73,16 @@ async def list_current_user_datasets(
     db: AsyncSession = Depends(get_async_db),
     workspace_id: Optional[UUID] = None,
     current_user: User = Security(auth.get_current_user),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
 ):
     await authorize(current_user, DatasetPolicy.list(workspace_id))
 
     if not workspace_id:
         if current_user.is_owner:
             dataset_list = await datasets.list_datasets(db)
+            await telemetry_client.track_resource_size(
+                crud_action="read", setting_name="dataset", count=len(dataset_list)
+            )
         else:
             await current_user.awaitable_attrs.datasets
             dataset_list = current_user.datasets
@@ -90,7 +94,10 @@ async def list_current_user_datasets(
 
 @router.get("/datasets/{dataset_id}/fields", response_model=Fields)
 async def list_dataset_fields(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.fields)])
 
@@ -101,7 +108,10 @@ async def list_dataset_fields(
 
 @router.get("/datasets/{dataset_id}/vectors-settings", response_model=VectorsSettings)
 async def list_dataset_vector_settings(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.vectors_settings)])
 
@@ -112,7 +122,10 @@ async def list_dataset_vector_settings(
 
 @router.get("/me/datasets/{dataset_id}/metadata-properties", response_model=MetadataProperties)
 async def list_current_user_dataset_metadata_properties(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.metadata_properties)])
 
@@ -127,7 +140,10 @@ async def list_current_user_dataset_metadata_properties(
 
 @router.get("/datasets/{dataset_id}", response_model=DatasetSchema)
 async def get_dataset(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id)
 
@@ -184,18 +200,24 @@ async def get_dataset_users_progress(
 async def create_dataset(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_create: DatasetCreate,
     current_user: User = Security(auth.get_current_user),
 ):
     await authorize(current_user, DatasetPolicy.create(dataset_create.workspace_id))
 
-    return await datasets.create_dataset(db, dataset_create.dict())
+    dataset = await datasets.create_dataset(db, dataset_create.dict())
+
+    await telemetry_client.track_crud_dataset(crud_action="create", dataset=dataset)
+
+    return dataset
 
 
 @router.post("/datasets/{dataset_id}/fields", status_code=status.HTTP_201_CREATED, response_model=Field)
 async def create_dataset_field(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     field_create: FieldCreate,
     current_user: User = Security(auth.get_current_user),
@@ -204,7 +226,11 @@ async def create_dataset_field(
 
     await authorize(current_user, DatasetPolicy.create_field(dataset))
 
-    return await datasets.create_field(db, dataset, field_create)
+    field = await datasets.create_field(db, dataset, field_create)
+
+    await telemetry_client.track_crud_dataset_setting(crud_action="create", setting_name="fields", setting=field)
+
+    return field
 
 
 @router.post(
@@ -214,6 +240,7 @@ async def create_dataset_metadata_property(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     metadata_property_create: MetadataPropertyCreate,
     current_user: User = Security(auth.get_current_user),
@@ -222,7 +249,13 @@ async def create_dataset_metadata_property(
 
     await authorize(current_user, DatasetPolicy.create_metadata_property(dataset))
 
-    return await datasets.create_metadata_property(db, search_engine, dataset, metadata_property_create)
+    metadata_property = await datasets.create_metadata_property(db, search_engine, dataset, metadata_property_create)
+
+    await telemetry_client.track_crud_dataset_setting(
+        crud_action="create", setting_name="metadata_properties", setting=metadata_property
+    )
+
+    return metadata_property
 
 
 @router.post(
@@ -232,6 +265,7 @@ async def create_dataset_vector_settings(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     vector_settings_create: VectorSettingsCreate,
     current_user: User = Security(auth.get_current_user),
@@ -240,15 +274,20 @@ async def create_dataset_vector_settings(
 
     await authorize(current_user, DatasetPolicy.create_vector_settings(dataset))
 
-    return await datasets.create_vector_settings(db, search_engine, dataset, vector_settings_create)
+    vector_setting = await datasets.create_vector_settings(db, search_engine, dataset, vector_settings_create)
+
+    await telemetry_client.track_crud_dataset_setting(
+        crud_action="create", setting_name="vectors_settings", setting=vector_setting
+    )
+
+    return vector_setting
 
 
 @router.put("/datasets/{dataset_id}/publish", response_model=DatasetSchema)
 async def publish_dataset(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
-    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     current_user: User = Security(auth.get_current_user),
 ) -> Dataset:
@@ -265,14 +304,7 @@ async def publish_dataset(
 
     await authorize(current_user, DatasetPolicy.publish(dataset))
 
-    dataset = await datasets.publish_dataset(db, search_engine, dataset)
-
-    telemetry_client.track_data(
-        action="PublishedDataset",
-        data={"questions": list(set([question.settings["type"] for question in dataset.questions]))},
-    )
-
-    return dataset
+    return await datasets.publish_dataset(db, search_engine, dataset)
 
 
 @router.delete("/datasets/{dataset_id}", response_model=DatasetSchema)

argilla-server/src/argilla_server/api/handlers/v1/datasets/questions.py

@@ -25,13 +25,17 @@
 from argilla_server.database import get_async_db
 from argilla_server.models import Dataset, User
 from argilla_server.security import auth
+from argilla_server.telemetry import TelemetryClient, get_telemetry_client
 
 router = APIRouter()
 
 
 @router.get("/datasets/{dataset_id}/questions", response_model=Questions)
 async def list_dataset_questions(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.questions)])
 
@@ -44,6 +48,7 @@ async def list_dataset_questions(
 async def create_dataset_question(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     question_create: QuestionCreate,
     current_user: User = Security(auth.get_current_user),
@@ -61,4 +66,8 @@ async def create_dataset_question(
 
     await authorize(current_user, DatasetPolicy.create_question(dataset))
 
-    return await questions.create_question(db, dataset, question_create)
+    question = await questions.create_question(db, dataset, question_create)
+
+    await telemetry_client.track_crud_dataset_setting(crud_action="create", setting_name="questions", setting=question)
+
+    return question

argilla-server/src/argilla_server/api/handlers/v1/datasets/records.py

@@ -12,7 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import re
 from typing import Any, Dict, List, Optional, Tuple, Union
 from uuid import UUID
 
@@ -46,7 +45,7 @@
 )
 from argilla_server.contexts import datasets, search
 from argilla_server.database import get_async_db
-from argilla_server.enums import RecordSortField, ResponseStatusFilter, SortOrder
+from argilla_server.enums import RecordSortField, ResponseStatusFilter
 from argilla_server.errors.future import MissingVectorError, NotFoundError, UnprocessableEntityError
 from argilla_server.errors.future.base_errors import MISSING_VECTOR_ERROR_CODE
 from argilla_server.models import Dataset, Field, Record, User, VectorSettings
@@ -58,7 +57,6 @@
     get_search_engine,
 )
 from argilla_server.security import auth
-from argilla_server.telemetry import TelemetryClient, get_telemetry_client
 from argilla_server.utils import parse_query_param, parse_uuids
 
 LIST_DATASET_RECORDS_LIMIT_DEFAULT = 50
@@ -305,7 +303,6 @@ async def search_current_user_dataset_records(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
-    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     body: SearchRecordsQuery,
     include: Optional[RecordIncludeParam] = Depends(parse_record_include_param),