maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Osquery] Fix 7.16.0 BC3 issues (#117105) (#117157)

Browse source 
Co-authored-by: Patryk Kopyciński <patryk.kopycinski@elastic.co>
This commit is contained in:
Kibana Machine 2021-11-02 16:49:43 -04:00 committed by GitHub
parent 8a55fa4fdc
commit e982d1023a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 741 additions and 486 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/osquery/common/schemas/common/schemas.ts
View file

@ -55,8 +55,9 @@ export type SavedQueryIdOrUndefined = t.TypeOf<typeof savedQueryIdOrUndefined>;
export const ecsMapping = t.record(
t.string,
t.type({
t.partial({
field: t.string,
value: t.string,
})
);
export type ECSMapping = t.TypeOf<typeof ecsMapping>;

48
x-pack/plugins/osquery/public/action_results/action_results_summary.tsx
View file

@ -13,7 +13,6 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { AgentIdToName } from '../agents/agent_id_to_name';
import { useActionResults } from './use_action_results';
import { useAllResults } from '../results/use_all_results';
import { Direction } from '../../common/search_strategy';
import { useActionResultsPrivileges } from './use_action_privileges';
@ -70,38 +69,8 @@ const ActionResultsSummaryComponent: React.FC<ActionResultsSummaryProps> = ({
});
}
const { data: logsResults } = useAllResults({
actionId,
activePage: pageIndex,
limit: pageSize,
sort: [
{
field: '@timestamp',
direction: Direction.asc,
},
],
isLive,
skip: !hasActionResultsPrivileges,
});
const renderAgentIdColumn = useCallback((agentId) => <AgentIdToName agentId={agentId} />, []);
const renderRowsColumn = useCallback(
(_, item) => {
if (!logsResults) return '-';
const agentId = item.fields.agent_id[0];
return (
// @ts-expect-error update types
logsResults?.rawResponse?.aggregations?.count_by_agent_id?.buckets?.find(
// @ts-expect-error update types
(bucket) => bucket.key === agentId
)?.doc_count ?? '-'
);
},
[logsResults]
);
const renderRowsColumn = useCallback((rowsCount) => rowsCount ?? '-', []);
const renderStatusColumn = useCallback(
(_, item) => {
if (!item.fields.completed_at) {
@ -145,7 +114,7 @@ const ActionResultsSummaryComponent: React.FC<ActionResultsSummaryProps> = ({
render: renderAgentIdColumn,
},
{
field: 'fields.rows[0]',
field: '_source.action_response.osquery.count',
name: i18n.translate(
'xpack.osquery.liveQueryActionResults.table.resultRowsNumberColumnTitle',
{
@ -177,18 +146,9 @@ const ActionResultsSummaryComponent: React.FC<ActionResultsSummaryProps> = ({
setIsLive(() => {
if (!agentIds?.length || expired) return false;
const uniqueAgentsRepliedCount =
// @ts-expect-error update types
logsResults?.rawResponse.aggregations?.unique_agents.value ?? 0;
return !!(uniqueAgentsRepliedCount !== agentIds?.length - aggregations.failed);
return !!(aggregations.totalResponded !== agentIds?.length);
});
}, [
agentIds?.length,
aggregations.failed,
expired,
logsResults?.rawResponse.aggregations?.unique_agents,
]);
}, [agentIds?.length, aggregations.totalResponded, expired]);
return edges.length ? (
<EuiInMemoryTable loading={isLive} items={edges} columns={columns} pagination={pagination} />

4
x-pack/plugins/osquery/public/action_results/use_action_results.ts
View file

@ -84,6 +84,9 @@ export const useActionResults = ({
const totalResponded =
// @ts-expect-error update types
responseData.rawResponse?.aggregations?.aggs.responses_by_action_id?.doc_count ?? 0;
const totalRowCount =
// @ts-expect-error update types
responseData.rawResponse?.aggregations?.aggs.responses_by_action_id?.rows_count?.value ?? 0;
const aggsBuckets =
// @ts-expect-error update types
responseData.rawResponse?.aggregations?.aggs.responses_by_action_id?.responses.buckets;
@ -100,6 +103,7 @@ export const useActionResults = ({
...responseData,
edges: reverse(uniqBy('fields.agent_id[0]', flatten([responseData.edges, previousEdges]))),
aggregations: {
totalRowCount,
totalResponded,
// @ts-expect-error update types
successful: aggsBuckets?.find((bucket) => bucket.key === 'success')?.doc_count ?? 0,

2
x-pack/plugins/osquery/public/common/schemas/ecs/v1.12.1.json
View file

File diff suppressed because one or more lines are too long

20
x-pack/plugins/osquery/public/fleet_integration/osquery_managed_policy_create_import_extension.tsx
View file

@ -165,16 +165,6 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo<
defaultValue: {
config: JSON.stringify(get(newPolicy, 'inputs[0].config.osquery.value', {}), null, 2),
},
serializer: (formData) => {
let config;
try {
// @ts-expect-error update types
config = JSON.parse(formData.config);
} catch (e) {
config = {};
}
return { config };
},
schema: {
config: {
label: i18n.translate('xpack.osquery.fleetIntegration.osqueryConfig.configFieldLabel', {
@ -243,10 +233,16 @@ export const OsqueryManagedPolicyCreateImportExtension = React.memo<
if (isValid === undefined) return;
const updatedPolicy = produce(newPolicy, (draft) => {
if (isEmpty(config)) {
let parsedConfig;
try {
parsedConfig = JSON.parse(config);
// eslint-disable-next-line no-empty
} catch (e) {}
if (isEmpty(parsedConfig)) {
unset(draft, 'inputs[0].config');
} else {
set(draft, 'inputs[0].config.osquery.value', config);
set(draft, 'inputs[0].config.osquery.value', parsedConfig);
}
return draft;
});

7
x-pack/plugins/osquery/public/packs/form/index.tsx
View file

@ -98,14 +98,17 @@ const PackFormComponent: React.FC<PackFormProps> = ({ defaultValue, editMode = f
description: {
type: FIELD_TYPES.TEXT,
label: i18n.translate('xpack.osquery.pack.form.descriptionFieldLabel', {
defaultMessage: 'Description',
defaultMessage: 'Description (optional)',
}),
},
policy_ids: {
defaultValue: [],
type: FIELD_TYPES.COMBO_BOX,
label: i18n.translate('xpack.osquery.pack.form.agentPoliciesFieldLabel', {
defaultMessage: 'Agent policies',
defaultMessage: 'Agent policies (optional)',
}),
helpText: i18n.translate('xpack.osquery.pack.form.agentPoliciesFieldHelpText', {
defaultMessage: 'Queries in this pack are scheduled for agents in the selected policies.',
}),
},
enabled: {

142
x-pack/plugins/osquery/public/packs/pack_queries_status_table.tsx
View file

@ -22,6 +22,7 @@ import {
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage, FormattedDate, FormattedTime, FormattedRelative } from '@kbn/i18n/react';
import moment from 'moment-timezone';
import {
TypedLensByValueInput,
@ -29,7 +30,7 @@ import {
PieVisualizationState,
} from '../../../lens/public';
import { FilterStateStore, IndexPattern } from '../../../../../src/plugins/data/common';
import { useKibana, isModifiedEvent, isLeftClickEvent } from '../common/lib/kibana';
import { useKibana } from '../common/lib/kibana';
import { OsqueryManagerPackagePolicyInputStream } from '../../common/types';
import { ScheduledQueryErrorsTable } from './scheduled_query_errors_table';
import { usePackQueryLastResults } from './use_pack_query_last_results';
@ -207,8 +208,6 @@ const ViewResultsInLensActionComponent: React.FC<ViewResultsInDiscoverActionProp
const handleClick = useCallback(
(event) => {
const openInNewTab = !(!isModifiedEvent(event) && isLeftClickEvent(event));
event.preventDefault();
lensService?.navigateToPrefilledEditor(
@ -222,7 +221,7 @@ const ViewResultsInLensActionComponent: React.FC<ViewResultsInDiscoverActionProp
attributes: getLensAttributes(actionId, agentIds),
},
{
openInNewTab,
openInNewTab: true,
}
);
},
@ -337,7 +336,7 @@ const ViewResultsInDiscoverActionComponent: React.FC<ViewResultsInDiscoverAction
if (buttonType === ViewResultsActionButtonType.button) {
return (
<EuiButtonEmpty size="xs" iconType="discoverApp" href={discoverUrl}>
<EuiButtonEmpty size="xs" iconType="discoverApp" href={discoverUrl} target="_blank">
{VIEW_IN_DISCOVER}
</EuiButtonEmpty>
);
@ -378,6 +377,7 @@ interface ScheduledQueryLastResultsProps {
actionId: string;
queryId: string;
interval: number;
logsIndexPattern: IndexPattern | undefined;
toggleErrors: (payload: { queryId: string; interval: number }) => void;
expanded: boolean;
}
@ -386,12 +386,10 @@ const ScheduledQueryLastResults: React.FC<ScheduledQueryLastResultsProps> = ({
actionId,
queryId,
interval,
logsIndexPattern,
toggleErrors,
expanded,
}) => {
const data = useKibana().services.data;
const [logsIndexPattern, setLogsIndexPattern] = useState<IndexPattern | undefined>(undefined);
const { data: lastResultsData, isFetched } = usePackQueryLastResults({
actionId,
interval,
@ -409,15 +407,6 @@ const ScheduledQueryLastResults: React.FC<ScheduledQueryLastResultsProps> = ({
[queryId, interval, toggleErrors]
);
useEffect(() => {
const fetchLogsIndexPattern = async () => {
const indexPattern = await data.indexPatterns.find('logs-*');
setLogsIndexPattern(indexPattern[0]);
};
fetchLogsIndexPattern();
}, [data.indexPatterns]);
if (!isFetched || !errorsFetched) {
return <EuiLoadingSpinner />;
}
@ -518,6 +507,86 @@ const ScheduledQueryLastResults: React.FC<ScheduledQueryLastResultsProps> = ({
const getPackActionId = (actionId: string, packName: string) => `pack_${packName}_${actionId}`;
interface PackViewInActionProps {
item: {
id: string;
interval: number;
};
logsIndexPattern: IndexPattern | undefined;
packName: string;
agentIds?: string[];
}
const PackViewInDiscoverActionComponent: React.FC<PackViewInActionProps> = ({
item,
logsIndexPattern,
packName,
agentIds,
}) => {
const { id, interval } = item;
const actionId = getPackActionId(id, packName);
const { data: lastResultsData } = usePackQueryLastResults({
actionId,
interval,
logsIndexPattern,
});
const startDate = lastResultsData?.['@timestamp']
? moment(lastResultsData?.['@timestamp'][0]).subtract(interval, 'seconds').toISOString()
: `now-${interval}s`;
const endDate = lastResultsData?.['@timestamp']
? moment(lastResultsData?.['@timestamp'][0]).toISOString()
: 'now';
return (
<ViewResultsInDiscoverAction
actionId={actionId}
agentIds={agentIds}
buttonType={ViewResultsActionButtonType.icon}
startDate={startDate}
endDate={endDate}
mode={lastResultsData?.['@timestamp'][0] ? 'absolute' : 'relative'}
/>
);
};
const PackViewInDiscoverAction = React.memo(PackViewInDiscoverActionComponent);
const PackViewInLensActionComponent: React.FC<PackViewInActionProps> = ({
item,
logsIndexPattern,
packName,
agentIds,
}) => {
const { id, interval } = item;
const actionId = getPackActionId(id, packName);
const { data: lastResultsData } = usePackQueryLastResults({
actionId,
interval,
logsIndexPattern,
});
const startDate = lastResultsData?.['@timestamp']
? moment(lastResultsData?.['@timestamp'][0]).subtract(interval, 'seconds').toISOString()
: `now-${interval}s`;
const endDate = lastResultsData?.['@timestamp']
? moment(lastResultsData?.['@timestamp'][0]).toISOString()
: 'now';
return (
<ViewResultsInLensAction
actionId={actionId}
agentIds={agentIds}
buttonType={ViewResultsActionButtonType.icon}
startDate={startDate}
endDate={endDate}
mode={lastResultsData?.['@timestamp'][0] ? 'absolute' : 'relative'}
/>
);
};
const PackViewInLensAction = React.memo(PackViewInLensActionComponent);
interface PackQueriesStatusTableProps {
agentIds?: string[];
data: OsqueryManagerPackagePolicyInputStream[];
@ -533,6 +602,18 @@ const PackQueriesStatusTableComponent: React.FC<PackQueriesStatusTableProps> = (
Record<string, ReturnType<typeof ScheduledQueryExpandedContent>>
>({});
const indexPatterns = useKibana().services.data.indexPatterns;
const [logsIndexPattern, setLogsIndexPattern] = useState<IndexPattern | undefined>(undefined);
useEffect(() => {
const fetchLogsIndexPattern = async () => {
const indexPattern = await indexPatterns.find('logs-*');
setLogsIndexPattern(indexPattern[0]);
};
fetchLogsIndexPattern();
}, [indexPatterns]);
const renderQueryColumn = useCallback(
(query: string) => (
<EuiCodeBlock language="sql" fontSize="s" paddingSize="none" transparentBackground>
@ -564,6 +645,7 @@ const PackQueriesStatusTableComponent: React.FC<PackQueriesStatusTableProps> = (
const renderLastResultsColumn = useCallback(
(item) => (
<ScheduledQueryLastResults
logsIndexPattern={logsIndexPattern}
queryId={item.id}
actionId={getPackActionId(item.id, packName)}
interval={item.interval}
@ -571,35 +653,31 @@ const PackQueriesStatusTableComponent: React.FC<PackQueriesStatusTableProps> = (
expanded={!!itemIdToExpandedRowMap[item.id]}
/>
),
[itemIdToExpandedRowMap, packName, toggleErrors]
[itemIdToExpandedRowMap, packName, toggleErrors, logsIndexPattern]
);
const renderDiscoverResultsAction = useCallback(
(item) => (
<ViewResultsInDiscoverAction
actionId={getPackActionId(item.id, packName)}
<PackViewInDiscoverAction
item={item}
agentIds={agentIds}
buttonType={ViewResultsActionButtonType.icon}
startDate={`now-${item.interval * 2}s`}
endDate="now"
mode="relative"
logsIndexPattern={logsIndexPattern}
packName={packName}
/>
),
[agentIds, packName]
[agentIds, logsIndexPattern, packName]
);
const renderLensResultsAction = useCallback(
(item) => (
<ViewResultsInLensAction
actionId={getPackActionId(item.id, packName)}
<PackViewInLensAction
item={item}
agentIds={agentIds}
buttonType={ViewResultsActionButtonType.icon}
startDate={`now-${item.interval * 2}s`}
endDate="now"
mode="relative"
logsIndexPattern={logsIndexPattern}
packName={packName}
/>
),
[agentIds, packName]
[agentIds, logsIndexPattern, packName]
);
const getItemId = useCallback(

2
x-pack/plugins/osquery/public/packs/packs_table.tsx
View file

@ -126,7 +126,7 @@ const PacksTableComponent = () => {
{
field: 'policy_ids',
name: i18n.translate('xpack.osquery.packs.table.policyColumnTitle', {
defaultMessage: 'Policies',
defaultMessage: 'Scheduled policies',
}),
truncateText: true,
render: renderAgentPolicy,

916
x-pack/plugins/osquery/public/packs/queries/ecs_mapping_editor_field.tsx
View file

File diff suppressed because it is too large Load diff

19
x-pack/plugins/osquery/public/packs/use_pack_query_last_results.ts
View file

@ -6,6 +6,7 @@
*/
import { useQuery } from 'react-query';
import moment from 'moment-timezone';
import { IndexPattern } from '../../../../../src/plugins/data/common';
import { useKibana } from '../common/lib/kibana';
@ -46,13 +47,12 @@ export const usePackQueryLastResults = ({
});
const lastResultsResponse = await lastResultsSearchSource.fetch$().toPromise();
const timestamp = lastResultsResponse.rawResponse?.hits?.hits[0]?.fields?.['@timestamp'][0];
const responseId = lastResultsResponse.rawResponse?.hits?.hits[0]?._source?.response_id;
if (responseId) {
if (timestamp) {
const aggsSearchSource = await data.search.searchSource.create({
index: logsIndexPattern,
size: 0,
size: 1,
aggs: {
unique_agents: { cardinality: { field: 'agent.id' } },
},
@ -61,13 +61,16 @@ export const usePackQueryLastResults = ({
bool: {
filter: [
{
match_phrase: {
action_id: actionId,
range: {
'@timestamp': {
gte: moment(timestamp).subtract(interval, 'seconds').format(),
lte: moment(timestamp).format(),
},
},
},
{
match_phrase: {
response_id: responseId,
action_id: actionId,
},
},
],
@ -81,7 +84,7 @@ export const usePackQueryLastResults = ({
'@timestamp': lastResultsResponse.rawResponse?.hits?.hits[0]?.fields?.['@timestamp'],
// @ts-expect-error update types
uniqueAgentsCount: aggsResponse.rawResponse.aggregations?.unique_agents?.value,
docCount: aggsResponse.rawResponse?.hits?.total,
docCount: aggsResponse?.rawResponse?.hits?.total,
};
}

16
x-pack/plugins/osquery/public/results/results_table.tsx
View file

@ -291,19 +291,9 @@ const ResultsTableComponent: React.FC<ResultsTableComponentProps> = ({
setIsLive(() => {
if (!agentIds?.length || expired) return false;
const uniqueAgentsRepliedCount =
// @ts-expect-error-type
allResultsData?.rawResponse.aggregations?.unique_agents.value ?? 0;
return !!(uniqueAgentsRepliedCount !== agentIds?.length - aggregations.failed);
return !!(aggregations.totalResponded !== agentIds?.length);
}),
[
agentIds?.length,
aggregations.failed,
// @ts-expect-error-type
allResultsData?.rawResponse.aggregations?.unique_agents.value,
expired,
]
[agentIds?.length, aggregations.failed, aggregations.totalResponded, expired]
);
if (!hasActionResultsPrivileges) {
@ -328,7 +318,7 @@ const ResultsTableComponent: React.FC<ResultsTableComponentProps> = ({
<>
{isLive && <EuiProgress color="primary" size="xs" />}
{isFetched && !allResultsData?.edges.length ? (
{isFetched && !allResultsData?.edges.length && !aggregations?.totalRowCount ? (
<>
<EuiCallOut title={generateEmptyDataMessage(aggregations.totalResponded)} />
<EuiSpacer />

10
x-pack/plugins/osquery/public/routes/packs/list/index.tsx
View file

@ -27,6 +27,16 @@ const PacksPageComponent = () => {
</h1>
</EuiText>
</EuiFlexItem>
<EuiFlexItem>
<EuiText color="subdued">
<p>
<FormattedMessage
id="xpack.osquery.packList.pageSubtitle"
defaultMessage="Create packs to organize sets of queries and to schedule queries for agent policies."
/>
</p>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
),
[]

1
x-pack/plugins/osquery/public/saved_queries/use_saved_queries.ts
View file

@ -36,6 +36,7 @@ export const useSavedQueries = ({
toastMessage: error.body.message,
});
},
refetchOnWindowFocus: !!isLive,
}
);
};

1
x-pack/plugins/osquery/public/saved_queries/use_saved_query.ts
View file

@ -29,6 +29,7 @@ export const useSavedQuery = ({ savedQueryId }: UseSavedQueryProps) => {
() => http.get(`/internal/osquery/saved_query/${savedQueryId}`),
{
keepPreviousData: true,
refetchOnWindowFocus: false,
onSuccess: (data) => {
if (data.error) {
setErrorToast(data.error, {

2
x-pack/plugins/osquery/scripts/schema_formatter/ecs_formatter.ts
View file

@ -11,7 +11,7 @@ import path from 'path';
import { run } from '@kbn/dev-utils';
const ECS_COLUMN_SCHEMA_FIELDS = ['field', 'type', 'description'];
const ECS_COLUMN_SCHEMA_FIELDS = ['field', 'type', 'normalization', 'example', 'description'];
const RESTRICTED_FIELDS = [
'agent.name',

10
x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
View file

@ -53,6 +53,11 @@ export const savedQueryType: SavedObjectsType = {
hidden: false,
namespaceType: 'multiple-isolated',
mappings: savedQuerySavedObjectMappings,
management: {
defaultSearchField: 'id',
importableAndExportable: true,
getTitle: (savedObject) => savedObject.attributes.id,
},
};
export const packSavedObjectMappings: SavedObjectsType['mappings'] = {
@ -109,4 +114,9 @@ export const packType: SavedObjectsType = {
hidden: false,
namespaceType: 'multiple-isolated',
mappings: packSavedObjectMappings,
management: {
defaultSearchField: 'name',
importableAndExportable: true,
getTitle: (savedObject) => savedObject.attributes.name,
},
};

2
x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts
View file

@ -26,7 +26,7 @@ export const readSavedQueryRoute = (router: IRouter) => {
const savedObjectsClient = context.core.savedObjects.client;
const savedQuery = await savedObjectsClient.get<{
ecs_mapping: Array<{ field: string; value: string }>;
ecs_mapping: Array<{ key: string; value: Record<string, object> }>;
}>(savedQuerySavedObjectType, request.params.id);
if (savedQuery.attributes.ecs_mapping) {

3
x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts
View file

@ -34,7 +34,8 @@ export const updateSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp
schema.recordOf(
schema.string(),
schema.object({
field: schema.string(),
field: schema.maybe(schema.string()),
value: schema.maybe(schema.string()),
})
)
),

14
x-pack/plugins/osquery/server/routes/utils.ts
View file

@ -5,22 +5,24 @@
* 2.0.
*/
import { pick, reduce } from 'lodash';
import { reduce } from 'lodash';
export const convertECSMappingToArray = (ecsMapping: Record<string, object> | undefined) =>
ecsMapping
? Object.entries(ecsMapping).map((item) => ({
value: item[0],
...item[1],
key: item[0],
value: item[1],
}))
: undefined;
export const convertECSMappingToObject = (ecsMapping: Array<{ field: string; value: string }>) =>
export const convertECSMappingToObject = (
ecsMapping: Array<{ key: string; value: Record<string, object> }>
) =>
reduce(
ecsMapping,
(acc, value) => {
acc[value.value] = pick(value, 'field');
acc[value.key] = value.value;
return acc;
},
{} as Record<string, { field: string }>
{} as Record<string, { field?: string; value?: string }>
);

5
x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts
View file

@ -46,6 +46,11 @@ export const buildActionResultsQuery = ({
},
},
aggs: {
rows_count: {
sum: {
field: 'action_response.osquery.count',
},
},
responses: {
terms: {
script: {