Skip to content

Commit

Permalink
[Osquery] Refactor telemetry to use EBT (elastic#138221)
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrykkopycinski
patrykkopycinski authored Aug 9, 2022
1 parent 26a4783 commit 3de9eda
Show file tree
Hide file tree
Showing 29 changed files with 530 additions and 922 deletions.
15 changes: 15 additions & 0 deletions x-pack/plugins/osquery/server/common/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,18 @@ export interface PackSavedObjectAttributes {
}

export type PackSavedObject = SavedObject<PackSavedObjectAttributes>;

export interface SavedQuerySavedObjectAttributes {
id: string;
description: string | undefined;
query: string;
interval: number | string;
platform: string;
ecs_mapping?: Array<Record<string, unknown>>;
created_at: string;
created_by: string | undefined;
updated_at: string;
updated_by: string | undefined;
}

export type SavedQuerySavedObject = SavedObject<PackSavedObjectAttributes>;
19 changes: 19 additions & 0 deletions x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,27 @@ import {
savedQuerySavedObjectType,
packSavedObjectType,
packAssetSavedObjectType,
usageMetricSavedObjectType,
} from '../../../common/types';

export const usageMetricSavedObjectMappings: SavedObjectsType['mappings'] = {
properties: {
count: {
type: 'long',
},
errors: {
type: 'long',
},
},
};

export const usageMetricType: SavedObjectsType = {
name: usageMetricSavedObjectType,
hidden: false,
namespaceType: 'agnostic',
mappings: usageMetricSavedObjectMappings,
};

export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
properties: {
description: {
Expand Down
2 changes: 0 additions & 2 deletions x-pack/plugins/osquery/server/lib/telemetry/__mocks__/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ export const createMockTelemetryEventsSender = (
setup: jest.fn(),
start: jest.fn(),
stop: jest.fn(),
getClusterID: jest.fn(),
fetchTelemetryUrl: jest.fn(),
queueTelemetryEvents: jest.fn(),
processEvents: jest.fn(),
isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()),
sendIfDue: jest.fn(),
Expand Down
12 changes: 4 additions & 8 deletions x-pack/plugins/osquery/server/lib/telemetry/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,7 @@
* 2.0.
*/

export const TELEMETRY_MAX_BUFFER_SIZE = 100;

export const MAX_PACK_TELEMETRY_BATCH = 100;

export const TELEMETRY_CHANNEL_CONFIGS = 'osquery-configs';
export const TELEMETRY_CHANNEL_LIVE_QUERIES = 'osquery-live-queries-test';
export const TELEMETRY_CHANNEL_PACKS = 'osquery-packs';
export const TELEMETRY_CHANNEL_SAVED_QUERIES = 'osquery-saved-queries';
export const TELEMETRY_EBT_LIVE_QUERY_EVENT = 'osquery_live_query';
export const TELEMETRY_EBT_PACK_EVENT = 'osquery_pack';
export const TELEMETRY_EBT_SAVED_QUERY_EVENT = 'osquery_saved_query';
export const TELEMETRY_EBT_CONFIG_EVENT = 'osquery_config';
107 changes: 43 additions & 64 deletions x-pack/plugins/osquery/server/lib/telemetry/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,85 +5,64 @@
* 2.0.
*/

import moment from 'moment';
import { filter, find, isEmpty, pick, isString } from 'lodash';
import type { SavedObjectsFindResponse } from '@kbn/core/server';
import type { PackagePolicy } from '@kbn/fleet-plugin/common';
import type { ESClusterInfo, ESLicense, ListTemplate, TelemetryEvent } from './types';
import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import type {
PackSavedObjectAttributes,
SavedQuerySavedObjectAttributes,
} from '../../common/types';

/**
* Constructs the configs telemetry schema from a collection of config saved objects
*/
export const templateConfigs = (
configsData: PackagePolicy[],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
) =>
configsData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};

return {
...template,
...item,
};
});
export const templateConfigs = (configsData: PackagePolicy[]) =>
configsData.map((item) => ({
id: item.id,
version: item.package?.version,
enabled: item.enabled,
config: find(item.inputs, ['type', 'osquery'])?.config?.osquery.value,
}));

/**
* Constructs the packs telemetry schema from a collection of packs saved objects
*/
export const templatePacks = (
packsData: SavedObjectsFindResponse['saved_objects'],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
) =>
packsData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};
packsData: SavedObjectsFindResponse<PackSavedObjectAttributes>['saved_objects']
) => {
const nonEmptyQueryPacks = filter(packsData, (pack) => !isEmpty(pack.attributes.queries));

return {
...template,
id: item.id,
...(item.attributes as TelemetryEvent),
};
});
return nonEmptyQueryPacks.map((item) =>
pick(
{
name: item.attributes.name,
enabled: item.attributes.enabled,
queries: item.attributes.queries,
policies: (filter(item.references, ['type', AGENT_POLICY_SAVED_OBJECT_TYPE]), 'id')?.length,
prebuilt:
!!filter(item.references, ['type', 'osquery-pack-asset']) &&
item.attributes.version !== undefined,
},
['name', 'queries', 'policies', 'prebuilt', 'enabled']
)
);
};

/**
* Constructs the packs telemetry schema from a collection of packs saved objects
*/
export const templateSavedQueries = (
savedQueriesData: SavedObjectsFindResponse['saved_objects'],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
savedQueriesData: SavedObjectsFindResponse<SavedQuerySavedObjectAttributes>['saved_objects'],
prebuiltSavedQueryIds: string[]
) =>
savedQueriesData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};

return {
...template,
id: item.id,
...(item.attributes as TelemetryEvent),
};
});

/**
* Convert counter label list to kebab case
*
* @param label_list the list of labels to create standardized UsageCounter from
* @returns a string label for usage in the UsageCounter
*/
export function createUsageCounterLabel(labelList: string[]): string {
return labelList.join('-');
}
savedQueriesData.map((item) => ({
id: item.attributes.id,
query: item.attributes.query,
platform: item.attributes.platform,
interval: isString(item.attributes.interval)
? parseInt(item.attributes.interval, 10)
: item.attributes.interval,
...(!isEmpty(item.attributes.ecs_mapping) ? { ecs_mapping: item.attributes.ecs_mapping } : {}),
prebuilt: prebuiltSavedQueryIds.includes(item.id),
}));
64 changes: 15 additions & 49 deletions x-pack/plugins/osquery/server/lib/telemetry/receiver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,28 @@ import type {
import type {
AgentClient,
AgentPolicyServiceInterface,
PackageService,
PackagePolicyServiceInterface,
} from '@kbn/fleet-plugin/server';
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import { OSQUERY_INTEGRATION_NAME } from '../../../common';
import { packSavedObjectType, savedQuerySavedObjectType } from '../../../common/types';
import type { ESLicense, ESClusterInfo } from './types';
import type { OsqueryAppContextService } from '../osquery_app_context_services';
import type {
PackSavedObjectAttributes,
SavedQuerySavedObjectAttributes,
} from '../../common/types';
import { getPrebuiltSavedQueryIds } from '../../routes/saved_query/utils';

export class TelemetryReceiver {
// @ts-expect-error used as part of this
private readonly logger: Logger;
private agentClient?: AgentClient;
private agentPolicyService?: AgentPolicyServiceInterface;
private packageService?: PackageService;
private packagePolicyService?: PackagePolicyServiceInterface;
private esClient?: ElasticsearchClient;
private soClient?: SavedObjectsClientContract;
private clusterInfo?: ESClusterInfo;
private readonly max_records = 100;

constructor(logger: Logger) {
Expand All @@ -40,19 +46,15 @@ export class TelemetryReceiver {
public async start(core: CoreStart, osqueryContextService?: OsqueryAppContextService) {
this.agentClient = osqueryContextService?.getAgentService()?.asInternalUser;
this.agentPolicyService = osqueryContextService?.getAgentPolicyService();
this.packageService = osqueryContextService?.getPackageService();
this.packagePolicyService = osqueryContextService?.getPackagePolicyService();
this.esClient = core.elasticsearch.client.asInternalUser;
this.soClient =
core.savedObjects.createInternalRepository() as unknown as SavedObjectsClientContract;
this.clusterInfo = await this.fetchClusterInfo();
}

public getClusterInfo(): ESClusterInfo | undefined {
return this.clusterInfo;
}

public async fetchPacks() {
return this.soClient?.find({
return this.soClient?.find<PackSavedObjectAttributes>({
type: packSavedObjectType,
page: 1,
perPage: this.max_records,
Expand All @@ -62,7 +64,7 @@ export class TelemetryReceiver {
}

public async fetchSavedQueries() {
return this.soClient?.find({
return this.soClient?.find<SavedQuerySavedObjectAttributes>({
type: savedQuerySavedObjectType,
page: 1,
perPage: this.max_records,
Expand All @@ -83,6 +85,10 @@ export class TelemetryReceiver {
throw Error('elasticsearch client is unavailable: cannot retrieve fleet policy responses');
}

public async fetchPrebuiltSavedQueryIds() {
return getPrebuiltSavedQueryIds(this.packageService?.asInternalUser);
}

public async fetchFleetAgents() {
if (this.esClient === undefined || this.soClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve fleet policy responses');
Expand All @@ -105,44 +111,4 @@ export class TelemetryReceiver {

return this.agentPolicyService?.get(this.soClient, id);
}

public async fetchClusterInfo(): Promise<ESClusterInfo> {
if (this.esClient === undefined || this.esClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
}

return this.esClient.info();
}

public async fetchLicenseInfo(): Promise<ESLicense | undefined> {
if (this.esClient === undefined || this.esClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve license information');
}

try {
const ret = await this.esClient.transport.request<{ license: ESLicense }>({
method: 'GET',
path: '/_license',
querystring: {
local: true,
},
});

return ret.license;
} catch (err) {
this.logger.debug(`failed retrieving license: ${err}`);

return undefined;
}
}

public copyLicenseFields(lic: ESLicense) {
return {
uid: lic.uid,
status: lic.status,
type: lic.type,
...(lic.issued_to ? { issued_to: lic.issued_to } : {}),
...(lic.issuer ? { issuer: lic.issuer } : {}),
};
}
}
Loading

0 comments on commit 3de9eda

Please sign in to comment.