[FEAT] Add integration with huggingface_hub.utils.telemetry

.github/workflows/argilla-frontend.deploy-environment.yml

@@ -63,7 +63,7 @@ jobs:
             ADMIN_API_KEY=${{ steps.credentials.outputs.admin }}
             ANNOTATOR_PASSWORD=${{ steps.credentials.outputs.annotator }}
             ANNOTATOR_API_KEY=${{ steps.credentials.outputs.annotator }}
-            ARGILLA_ENABLE_TELEMETRY=0
+            HF_HUB_DISABLE_TELEMETRY=1
             API_BASE_URL=https://dev.argilla.io/
 
       - name: Post credentials in Slack

.github/workflows/argilla-server.yml

@@ -52,7 +52,7 @@ jobs:
           - 5432:5432
 
     env:
-      ARGILLA_ENABLE_TELEMETRY: 0
+      HF_HUB_DISABLE_TELEMETRY: 1
 
     steps:
       - name: Checkout Code 🛎

argilla-server/pyproject.toml

@@ -57,6 +57,8 @@ dependencies = [
     "typer >= 0.6.0, < 0.10.0", # spaCy only supports typer<0.10.0
     "packaging>=23.2",
     "psycopg2-binary>=2.9.9",
+    # For Telemetry
+    "huggingface_hub>=0.13,<1",
 ]
 
 [project.optional-dependencies]
@@ -106,7 +108,7 @@ log_format = "%(asctime)s %(name)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_cli = "True"
 testpaths = ["tests"]
-env = ["ARGILLA_ENABLE_TELEMETRY=0"]
+env = ["HF_HUB_DISABLE_TELEMETRY=1"]
 
 [tool.coverage.run]
 concurrency = ["greenlet", "thread", "multiprocessing"]

argilla-server/src/argilla_server/_app.py

@@ -174,10 +174,14 @@ def show_telemetry_warning():
             "    https://docs.argilla.io/latest/reference/argilla-server/telemetry/\n\n"
             "Telemetry is currently enabled. If you want to disable it, you can configure\n"
             "the environment variable before relaunching the server:\n\n"
-            f'{"#set ARGILLA_ENABLE_TELEMETRY=0" if os.name == "nt" else "$>export ARGILLA_ENABLE_TELEMETRY=0"}'
+            f'{"#set HF_HUB_DISABLE_TELEMETRY=1" if os.name == "nt" else "$>export HF_HUB_DISABLE_TELEMETRY=1"}'
         )
         _LOGGER.warning(message)
 
+    message += "\n\n    "
+    message += "#set HF_HUB_DISABLE_TELEMETRY=1" if os.name == "nt" else "$>export HF_HUB_DISABLE_TELEMETRY=1"
+    message += "\n"
+
 
 async def _create_oauth_allowed_workspaces(db: AsyncSession):
     from argilla_server.security.settings import settings as security_settings

argilla-server/src/argilla_server/api/handlers/v1/datasets/datasets.py

@@ -22,14 +22,14 @@
 from argilla_server.api.policies.v1 import DatasetPolicy, MetadataPropertyPolicy, authorize, is_authorized
 from argilla_server.api.schemas.v1.datasets import (
     Dataset as DatasetSchema,
-    UsersProgress,
 )
 from argilla_server.api.schemas.v1.datasets import (
     DatasetCreate,
     DatasetMetrics,
     DatasetProgress,
     Datasets,
     DatasetUpdate,
+    UsersProgress,
 )
 from argilla_server.api.schemas.v1.fields import Field, FieldCreate, Fields
 from argilla_server.api.schemas.v1.metadata_properties import (
@@ -71,6 +71,7 @@ async def _filter_metadata_properties_by_policy(
 async def list_current_user_datasets(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     workspace_id: Optional[UUID] = None,
     current_user: User = Security(auth.get_current_user),
 ):
@@ -85,34 +86,64 @@ async def list_current_user_datasets(
     else:
         dataset_list = await datasets.list_datasets_by_workspace_id(db, workspace_id)
 
+    await telemetry_client.track_crud_dataset(action="me/list", dataset=None, count=len(dataset_list))
+
     return Datasets(items=dataset_list)
 
 
 @router.get("/datasets/{dataset_id}/fields", response_model=Fields)
 async def list_dataset_fields(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.fields)])
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
+    for field in dataset.fields:
+        await telemetry_client.track_crud_dataset_setting(
+            action="read", dataset=dataset, setting_name="fields", setting=field
+        )
+    await telemetry_client.track_crud_dataset_setting(
+        action="list", dataset=dataset, setting_name="fields", count=len(dataset.fields)
+    )
+
     return Fields(items=dataset.fields)
 
 
 @router.get("/datasets/{dataset_id}/vectors-settings", response_model=VectorsSettings)
 async def list_dataset_vector_settings(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.vectors_settings)])
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
+    for vectors_setting in dataset.vectors_settings:
+        await telemetry_client.track_crud_dataset_setting(
+            action="read", dataset=dataset, setting_name="vectors_settings", setting=vectors_setting
+        )
+    await telemetry_client.track_crud_dataset_setting(
+        action="list", dataset=dataset, setting_name="vectors_settings", count=len(dataset.vectors_settings)
+    )
+
     return VectorsSettings(items=dataset.vectors_settings)
 
 
 @router.get("/me/datasets/{dataset_id}/metadata-properties", response_model=MetadataProperties)
 async def list_current_user_dataset_metadata_properties(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id, options=[selectinload(Dataset.metadata_properties)])
 
@@ -122,17 +153,31 @@ async def list_current_user_dataset_metadata_properties(
         current_user, dataset.metadata_properties
     )
 
+    for metadata_property in filtered_metadata_properties:
+        await telemetry_client.track_crud_dataset_setting(
+            action="read", dataset=dataset, setting_name="me/metadata_properties", setting=metadata_property
+        )
+    await telemetry_client.track_crud_dataset_setting(
+        action="list", dataset=dataset, setting_name="me/metadata_properties", count=len(filtered_metadata_properties)
+    )
+
     return MetadataProperties(items=filtered_metadata_properties)
 
 
 @router.get("/datasets/{dataset_id}", response_model=DatasetSchema)
 async def get_dataset(
-    *, db: AsyncSession = Depends(get_async_db), dataset_id: UUID, current_user: User = Security(auth.get_current_user)
+    *,
+    db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
+    dataset_id: UUID,
+    current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id)
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
+    await telemetry_client.track_crud_dataset(action="read", dataset=dataset)
+
     return dataset
 
 
@@ -184,18 +229,24 @@ async def get_dataset_users_progress(
 async def create_dataset(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_create: DatasetCreate,
     current_user: User = Security(auth.get_current_user),
 ):
     await authorize(current_user, DatasetPolicy.create(dataset_create.workspace_id))
 
-    return await datasets.create_dataset(db, dataset_create.dict())
+    dataset = await datasets.create_dataset(db, dataset_create.dict())
+
+    await telemetry_client.track_crud_dataset(action="create", dataset=dataset)
+
+    return dataset
 
 
 @router.post("/datasets/{dataset_id}/fields", status_code=status.HTTP_201_CREATED, response_model=Field)
 async def create_dataset_field(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     field_create: FieldCreate,
     current_user: User = Security(auth.get_current_user),
@@ -204,7 +255,13 @@ async def create_dataset_field(
 
     await authorize(current_user, DatasetPolicy.create_field(dataset))
 
-    return await datasets.create_field(db, dataset, field_create)
+    field = await datasets.create_field(db, dataset, field_create)
+
+    await telemetry_client.track_crud_dataset_setting(
+        action="create", setting_name="fields", dataset=dataset, setting=field
+    )
+
+    return field
 
 
 @router.post(
@@ -214,6 +271,7 @@ async def create_dataset_metadata_property(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     metadata_property_create: MetadataPropertyCreate,
     current_user: User = Security(auth.get_current_user),
@@ -222,7 +280,13 @@ async def create_dataset_metadata_property(
 
     await authorize(current_user, DatasetPolicy.create_metadata_property(dataset))
 
-    return await datasets.create_metadata_property(db, search_engine, dataset, metadata_property_create)
+    metadata_property = await datasets.create_metadata_property(db, search_engine, dataset, metadata_property_create)
+
+    await telemetry_client.track_crud_dataset_setting(
+        action="create", setting_name="metadata_properties", dataset=dataset, setting=metadata_property
+    )
+
+    return metadata_property
 
 
 @router.post(
@@ -232,6 +296,7 @@ async def create_dataset_vector_settings(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     vector_settings_create: VectorSettingsCreate,
     current_user: User = Security(auth.get_current_user),
@@ -240,7 +305,13 @@ async def create_dataset_vector_settings(
 
     await authorize(current_user, DatasetPolicy.create_vector_settings(dataset))
 
-    return await datasets.create_vector_settings(db, search_engine, dataset, vector_settings_create)
+    vector_setting = await datasets.create_vector_settings(db, search_engine, dataset, vector_settings_create)
+
+    await telemetry_client.track_crud_dataset_setting(
+        action="create", setting_name="vectors_settings", dataset=dataset, setting=vector_setting
+    )
+
+    return vector_setting
 
 
 @router.put("/datasets/{dataset_id}/publish", response_model=DatasetSchema)
@@ -267,10 +338,7 @@ async def publish_dataset(
 
     dataset = await datasets.publish_dataset(db, search_engine, dataset)
 
-    telemetry_client.track_data(
-        action="PublishedDataset",
-        data={"questions": list(set([question.settings["type"] for question in dataset.questions]))},
-    )
+    await telemetry_client.track_crud_dataset(action="create", dataset=dataset)
 
     return dataset
 
@@ -280,20 +348,26 @@ async def delete_dataset(
     *,
     db: AsyncSession = Depends(get_async_db),
     search_engine: SearchEngine = Depends(get_search_engine),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id)
 
     await authorize(current_user, DatasetPolicy.delete(dataset))
 
-    return await datasets.delete_dataset(db, search_engine, dataset)
+    dataset = await datasets.delete_dataset(db, search_engine, dataset)
+
+    await telemetry_client.track_crud_dataset(action="delete", dataset=dataset)
+
+    return dataset
 
 
 @router.patch("/datasets/{dataset_id}", response_model=DatasetSchema)
 async def update_dataset(
     *,
     db: AsyncSession = Depends(get_async_db),
+    telemetry_client: TelemetryClient = Depends(get_telemetry_client),
     dataset_id: UUID,
     dataset_update: DatasetUpdate,
     current_user: User = Security(auth.get_current_user),
@@ -302,4 +376,8 @@ async def update_dataset(
 
     await authorize(current_user, DatasetPolicy.update(dataset))
 
-    return await datasets.update_dataset(db, dataset, dataset_update.dict(exclude_unset=True))
+    dataset = await datasets.update_dataset(db, dataset, dataset_update.dict(exclude_unset=True))
+
+    await telemetry_client.track_crud_dataset(action="update", dataset=dataset)
+
+    return dataset