[Ingest Manager] Fix and improve agent status (#71009)
This commit is contained in:
parent
237b2f66ed
commit
c8e675492b
|
@ -5,63 +5,52 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
AGENT_TYPE_TEMPORARY,
|
|
||||||
AGENT_POLLING_THRESHOLD_MS,
|
AGENT_POLLING_THRESHOLD_MS,
|
||||||
AGENT_TYPE_PERMANENT,
|
AGENT_TYPE_PERMANENT,
|
||||||
AGENT_TYPE_EPHEMERAL,
|
|
||||||
AGENT_SAVED_OBJECT_TYPE,
|
AGENT_SAVED_OBJECT_TYPE,
|
||||||
} from '../constants';
|
} from '../constants';
|
||||||
import { Agent, AgentStatus } from '../types';
|
import { Agent, AgentStatus } from '../types';
|
||||||
|
|
||||||
export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
|
export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
|
||||||
const { type, last_checkin: lastCheckIn } = agent;
|
const { last_checkin: lastCheckIn } = agent;
|
||||||
const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
|
|
||||||
const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
|
|
||||||
const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
|
|
||||||
if (!agent.active) {
|
if (!agent.active) {
|
||||||
return 'inactive';
|
return 'inactive';
|
||||||
}
|
}
|
||||||
|
if (!agent.last_checkin) {
|
||||||
|
return 'enrolling';
|
||||||
|
}
|
||||||
if (agent.unenrollment_started_at && !agent.unenrolled_at) {
|
if (agent.unenrollment_started_at && !agent.unenrolled_at) {
|
||||||
return 'unenrolling';
|
return 'unenrolling';
|
||||||
}
|
}
|
||||||
if (agent.current_error_events.length > 0) {
|
|
||||||
|
const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
|
||||||
|
const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
|
||||||
|
const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
|
||||||
|
|
||||||
|
if (agent.last_checkin_status === 'error') {
|
||||||
return 'error';
|
return 'error';
|
||||||
}
|
}
|
||||||
switch (type) {
|
if (agent.last_checkin_status === 'degraded') {
|
||||||
case AGENT_TYPE_PERMANENT:
|
return 'degraded';
|
||||||
if (intervalsSinceLastCheckIn >= 4) {
|
|
||||||
return 'error';
|
|
||||||
}
|
|
||||||
case AGENT_TYPE_TEMPORARY:
|
|
||||||
if (intervalsSinceLastCheckIn >= 3) {
|
|
||||||
return 'offline';
|
|
||||||
}
|
|
||||||
case AGENT_TYPE_EPHEMERAL:
|
|
||||||
if (intervalsSinceLastCheckIn >= 3) {
|
|
||||||
return 'inactive';
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if (intervalsSinceLastCheckIn >= 4) {
|
||||||
|
return 'offline';
|
||||||
|
}
|
||||||
|
|
||||||
return 'online';
|
return 'online';
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildKueryForOnlineAgents() {
|
export function buildKueryForOnlineAgents() {
|
||||||
return `(${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()})`;
|
||||||
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
|
||||||
}s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
|
||||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
|
||||||
}s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_EPHEMERAL} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
|
||||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
|
||||||
}s)`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildKueryForOfflineAgents() {
|
|
||||||
return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
|
||||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
|
||||||
}s`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildKueryForErrorAgents() {
|
export function buildKueryForErrorAgents() {
|
||||||
return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
return `( ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded )`;
|
||||||
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
}
|
||||||
}s`;
|
|
||||||
|
export function buildKueryForOfflineAgents() {
|
||||||
|
return `((${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
||||||
|
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||||
|
}s) AND not ( ${buildKueryForErrorAgents()} ))`;
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,7 +11,16 @@ export type AgentType =
|
||||||
| typeof AGENT_TYPE_PERMANENT
|
| typeof AGENT_TYPE_PERMANENT
|
||||||
| typeof AGENT_TYPE_TEMPORARY;
|
| typeof AGENT_TYPE_TEMPORARY;
|
||||||
|
|
||||||
export type AgentStatus = 'offline' | 'error' | 'online' | 'inactive' | 'warning' | 'unenrolling';
|
export type AgentStatus =
|
||||||
|
| 'offline'
|
||||||
|
| 'error'
|
||||||
|
| 'online'
|
||||||
|
| 'inactive'
|
||||||
|
| 'warning'
|
||||||
|
| 'enrolling'
|
||||||
|
| 'unenrolling'
|
||||||
|
| 'degraded';
|
||||||
|
|
||||||
export type AgentActionType = 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE' | 'UNENROLL';
|
export type AgentActionType = 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE' | 'UNENROLL';
|
||||||
export interface NewAgentAction {
|
export interface NewAgentAction {
|
||||||
type: AgentActionType;
|
type: AgentActionType;
|
||||||
|
@ -82,6 +91,7 @@ interface AgentBase {
|
||||||
config_id?: string;
|
config_id?: string;
|
||||||
config_revision?: number | null;
|
config_revision?: number | null;
|
||||||
last_checkin?: string;
|
last_checkin?: string;
|
||||||
|
last_checkin_status?: 'error' | 'online' | 'degraded';
|
||||||
user_provided_metadata: AgentMetadata;
|
user_provided_metadata: AgentMetadata;
|
||||||
local_metadata: AgentMetadata;
|
local_metadata: AgentMetadata;
|
||||||
}
|
}
|
||||||
|
|
|
@ -47,6 +47,7 @@ export interface PostAgentCheckinRequest {
|
||||||
agentId: string;
|
agentId: string;
|
||||||
};
|
};
|
||||||
body: {
|
body: {
|
||||||
|
status?: 'online' | 'error' | 'degraded';
|
||||||
local_metadata?: Record<string, any>;
|
local_metadata?: Record<string, any>;
|
||||||
events?: NewAgentEvent[];
|
events?: NewAgentEvent[];
|
||||||
};
|
};
|
||||||
|
|
|
@ -178,11 +178,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (selectedStatus.length) {
|
if (selectedStatus.length) {
|
||||||
if (kuery) {
|
const kueryStatus = selectedStatus
|
||||||
kuery = `(${kuery}) and`;
|
|
||||||
}
|
|
||||||
|
|
||||||
kuery = selectedStatus
|
|
||||||
.map((status) => {
|
.map((status) => {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case 'online':
|
case 'online':
|
||||||
|
@ -196,6 +192,12 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
|
||||||
return '';
|
return '';
|
||||||
})
|
})
|
||||||
.join(' or ');
|
.join(' or ');
|
||||||
|
|
||||||
|
if (kuery) {
|
||||||
|
kuery = `(${kuery}) and ${kueryStatus}`;
|
||||||
|
} else {
|
||||||
|
kuery = kueryStatus;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const agentsRequest = useGetAgents(
|
const agentsRequest = useGetAgents(
|
||||||
|
|
|
@ -53,6 +53,22 @@ const Status = {
|
||||||
/>
|
/>
|
||||||
</EuiHealth>
|
</EuiHealth>
|
||||||
),
|
),
|
||||||
|
Degraded: (
|
||||||
|
<EuiHealth color="danger">
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.ingestManager.agentHealth.degradedStatusText"
|
||||||
|
defaultMessage="Degraded"
|
||||||
|
/>
|
||||||
|
</EuiHealth>
|
||||||
|
),
|
||||||
|
Enrolling: (
|
||||||
|
<EuiHealth color="warning">
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.ingestManager.agentHealth.enrollingStatusText"
|
||||||
|
defaultMessage="Enrolling"
|
||||||
|
/>
|
||||||
|
</EuiHealth>
|
||||||
|
),
|
||||||
Unenrolling: (
|
Unenrolling: (
|
||||||
<EuiHealth color="warning">
|
<EuiHealth color="warning">
|
||||||
<FormattedMessage
|
<FormattedMessage
|
||||||
|
@ -67,6 +83,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
|
||||||
switch (agent.status) {
|
switch (agent.status) {
|
||||||
case 'error':
|
case 'error':
|
||||||
return Status.Error;
|
return Status.Error;
|
||||||
|
case 'degraded':
|
||||||
|
return Status.Degraded;
|
||||||
case 'inactive':
|
case 'inactive':
|
||||||
return Status.Inactive;
|
return Status.Inactive;
|
||||||
case 'offline':
|
case 'offline':
|
||||||
|
@ -75,6 +93,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
|
||||||
return Status.Warning;
|
return Status.Warning;
|
||||||
case 'unenrolling':
|
case 'unenrolling':
|
||||||
return Status.Unenrolling;
|
return Status.Unenrolling;
|
||||||
|
case 'enrolling':
|
||||||
|
return Status.Enrolling;
|
||||||
default:
|
default:
|
||||||
return Status.Online;
|
return Status.Online;
|
||||||
}
|
}
|
||||||
|
|
|
@ -178,8 +178,11 @@ export const postAgentCheckinHandler: RequestHandler<
|
||||||
const { actions } = await AgentService.agentCheckin(
|
const { actions } = await AgentService.agentCheckin(
|
||||||
soClient,
|
soClient,
|
||||||
agent,
|
agent,
|
||||||
request.body.events || [],
|
{
|
||||||
request.body.local_metadata,
|
events: request.body.events || [],
|
||||||
|
localMetadata: request.body.local_metadata,
|
||||||
|
status: request.body.status,
|
||||||
|
},
|
||||||
{ signal }
|
{ signal }
|
||||||
);
|
);
|
||||||
const body: PostAgentCheckinResponse = {
|
const body: PostAgentCheckinResponse = {
|
||||||
|
|
|
@ -63,6 +63,7 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
|
||||||
config_id: { type: 'keyword' },
|
config_id: { type: 'keyword' },
|
||||||
last_updated: { type: 'date' },
|
last_updated: { type: 'date' },
|
||||||
last_checkin: { type: 'date' },
|
last_checkin: { type: 'date' },
|
||||||
|
last_checkin_status: { type: 'keyword' },
|
||||||
config_revision: { type: 'integer' },
|
config_revision: { type: 'integer' },
|
||||||
default_api_key_id: { type: 'keyword' },
|
default_api_key_id: { type: 'keyword' },
|
||||||
default_api_key: { type: 'binary', index: false },
|
default_api_key: { type: 'binary', index: false },
|
||||||
|
@ -310,6 +311,7 @@ export function registerEncryptedSavedObjects(
|
||||||
'config_id',
|
'config_id',
|
||||||
'last_updated',
|
'last_updated',
|
||||||
'last_checkin',
|
'last_checkin',
|
||||||
|
'last_checkin_status',
|
||||||
'config_revision',
|
'config_revision',
|
||||||
'config_newest_revision',
|
'config_newest_revision',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
|
|
|
@ -11,7 +11,6 @@ import {
|
||||||
AgentEvent,
|
AgentEvent,
|
||||||
AgentSOAttributes,
|
AgentSOAttributes,
|
||||||
AgentEventSOAttributes,
|
AgentEventSOAttributes,
|
||||||
AgentMetadata,
|
|
||||||
} from '../../../types';
|
} from '../../../types';
|
||||||
|
|
||||||
import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants';
|
import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants';
|
||||||
|
@ -21,20 +20,24 @@ import { getAgentActionsForCheckin } from '../actions';
|
||||||
export async function agentCheckin(
|
export async function agentCheckin(
|
||||||
soClient: SavedObjectsClientContract,
|
soClient: SavedObjectsClientContract,
|
||||||
agent: Agent,
|
agent: Agent,
|
||||||
events: NewAgentEvent[],
|
data: {
|
||||||
localMetadata?: any,
|
events: NewAgentEvent[];
|
||||||
|
localMetadata?: any;
|
||||||
|
status?: 'online' | 'error' | 'degraded';
|
||||||
|
},
|
||||||
options?: { signal: AbortSignal }
|
options?: { signal: AbortSignal }
|
||||||
) {
|
) {
|
||||||
const updateData: {
|
const updateData: Partial<AgentSOAttributes> = {};
|
||||||
local_metadata?: AgentMetadata;
|
const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, data.events);
|
||||||
current_error_events?: string;
|
|
||||||
} = {};
|
|
||||||
const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, events);
|
|
||||||
if (updatedErrorEvents) {
|
if (updatedErrorEvents) {
|
||||||
updateData.current_error_events = JSON.stringify(updatedErrorEvents);
|
updateData.current_error_events = JSON.stringify(updatedErrorEvents);
|
||||||
}
|
}
|
||||||
if (localMetadata) {
|
if (data.localMetadata) {
|
||||||
updateData.local_metadata = localMetadata;
|
updateData.local_metadata = data.localMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.status !== agent.last_checkin_status) {
|
||||||
|
updateData.last_checkin_status = data.status;
|
||||||
}
|
}
|
||||||
if (Object.keys(updateData).length > 0) {
|
if (Object.keys(updateData).length > 0) {
|
||||||
await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData);
|
await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData);
|
||||||
|
|
|
@ -59,7 +59,7 @@ export function agentCheckinStateConnectedAgentsFactory() {
|
||||||
const internalSOClient = getInternalUserSOClient();
|
const internalSOClient = getInternalUserSOClient();
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const updates: Array<SavedObjectsBulkUpdateObject<AgentSOAttributes>> = [
|
const updates: Array<SavedObjectsBulkUpdateObject<AgentSOAttributes>> = [
|
||||||
...connectedAgentsIds.values(),
|
...agentToUpdate.values(),
|
||||||
].map((agentId) => ({
|
].map((agentId) => ({
|
||||||
type: AGENT_SAVED_OBJECT_TYPE,
|
type: AGENT_SAVED_OBJECT_TYPE,
|
||||||
id: agentId,
|
id: agentId,
|
||||||
|
|
|
@ -33,6 +33,7 @@ describe('Agent status service', () => {
|
||||||
type: AGENT_TYPE_PERMANENT,
|
type: AGENT_TYPE_PERMANENT,
|
||||||
attributes: {
|
attributes: {
|
||||||
active: true,
|
active: true,
|
||||||
|
last_checkin: new Date().toISOString(),
|
||||||
local_metadata: {},
|
local_metadata: {},
|
||||||
user_provided_metadata: {},
|
user_provided_metadata: {},
|
||||||
},
|
},
|
||||||
|
@ -40,4 +41,36 @@ describe('Agent status service', () => {
|
||||||
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||||
expect(status).toEqual('online');
|
expect(status).toEqual('online');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should return enrolling when agent is active but never checkin', async () => {
|
||||||
|
const mockSavedObjectsClient = savedObjectsClientMock.create();
|
||||||
|
mockSavedObjectsClient.get = jest.fn().mockReturnValue({
|
||||||
|
id: 'id',
|
||||||
|
type: AGENT_TYPE_PERMANENT,
|
||||||
|
attributes: {
|
||||||
|
active: true,
|
||||||
|
local_metadata: {},
|
||||||
|
user_provided_metadata: {},
|
||||||
|
},
|
||||||
|
} as SavedObject<AgentSOAttributes>);
|
||||||
|
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||||
|
expect(status).toEqual('enrolling');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return unenrolling when agent is unenrolling', async () => {
|
||||||
|
const mockSavedObjectsClient = savedObjectsClientMock.create();
|
||||||
|
mockSavedObjectsClient.get = jest.fn().mockReturnValue({
|
||||||
|
id: 'id',
|
||||||
|
type: AGENT_TYPE_PERMANENT,
|
||||||
|
attributes: {
|
||||||
|
active: true,
|
||||||
|
last_checkin: new Date().toISOString(),
|
||||||
|
unenrollment_started_at: new Date().toISOString(),
|
||||||
|
local_metadata: {},
|
||||||
|
user_provided_metadata: {},
|
||||||
|
},
|
||||||
|
} as SavedObject<AgentSOAttributes>);
|
||||||
|
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||||
|
expect(status).toEqual('unenrolling');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -32,6 +32,9 @@ export const PostAgentCheckinRequestSchema = {
|
||||||
agentId: schema.string(),
|
agentId: schema.string(),
|
||||||
}),
|
}),
|
||||||
body: schema.object({
|
body: schema.object({
|
||||||
|
status: schema.maybe(
|
||||||
|
schema.oneOf([schema.literal('online'), schema.literal('error'), schema.literal('degraded')])
|
||||||
|
),
|
||||||
local_metadata: schema.maybe(schema.recordOf(schema.string(), schema.any())),
|
local_metadata: schema.maybe(schema.recordOf(schema.string(), schema.any())),
|
||||||
events: schema.maybe(schema.arrayOf(NewAgentEventSchema)),
|
events: schema.maybe(schema.arrayOf(NewAgentEventSchema)),
|
||||||
}),
|
}),
|
||||||
|
|
Loading…
Reference in a new issue