[Ingest Manager] Fix and improve agent status (#71009)
This commit is contained in:
parent
237b2f66ed
commit
c8e675492b
|
@ -5,63 +5,52 @@
|
|||
*/
|
||||
|
||||
import {
|
||||
AGENT_TYPE_TEMPORARY,
|
||||
AGENT_POLLING_THRESHOLD_MS,
|
||||
AGENT_TYPE_PERMANENT,
|
||||
AGENT_TYPE_EPHEMERAL,
|
||||
AGENT_SAVED_OBJECT_TYPE,
|
||||
} from '../constants';
|
||||
import { Agent, AgentStatus } from '../types';
|
||||
|
||||
export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
|
||||
const { type, last_checkin: lastCheckIn } = agent;
|
||||
const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
|
||||
const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
|
||||
const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
|
||||
const { last_checkin: lastCheckIn } = agent;
|
||||
|
||||
if (!agent.active) {
|
||||
return 'inactive';
|
||||
}
|
||||
if (!agent.last_checkin) {
|
||||
return 'enrolling';
|
||||
}
|
||||
if (agent.unenrollment_started_at && !agent.unenrolled_at) {
|
||||
return 'unenrolling';
|
||||
}
|
||||
if (agent.current_error_events.length > 0) {
|
||||
|
||||
const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
|
||||
const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
|
||||
const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
|
||||
|
||||
if (agent.last_checkin_status === 'error') {
|
||||
return 'error';
|
||||
}
|
||||
switch (type) {
|
||||
case AGENT_TYPE_PERMANENT:
|
||||
if (agent.last_checkin_status === 'degraded') {
|
||||
return 'degraded';
|
||||
}
|
||||
if (intervalsSinceLastCheckIn >= 4) {
|
||||
return 'error';
|
||||
}
|
||||
case AGENT_TYPE_TEMPORARY:
|
||||
if (intervalsSinceLastCheckIn >= 3) {
|
||||
return 'offline';
|
||||
}
|
||||
case AGENT_TYPE_EPHEMERAL:
|
||||
if (intervalsSinceLastCheckIn >= 3) {
|
||||
return 'inactive';
|
||||
}
|
||||
}
|
||||
|
||||
return 'online';
|
||||
}
|
||||
|
||||
export function buildKueryForOnlineAgents() {
|
||||
return `(${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
||||
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_EPHEMERAL} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
|
||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s)`;
|
||||
}
|
||||
|
||||
export function buildKueryForOfflineAgents() {
|
||||
return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
||||
(3 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s`;
|
||||
return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()})`;
|
||||
}
|
||||
|
||||
export function buildKueryForErrorAgents() {
|
||||
return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
||||
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s`;
|
||||
return `( ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded )`;
|
||||
}
|
||||
|
||||
export function buildKueryForOfflineAgents() {
|
||||
return `((${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
|
||||
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
|
||||
}s) AND not ( ${buildKueryForErrorAgents()} ))`;
|
||||
}
|
||||
|
|
|
@ -11,7 +11,16 @@ export type AgentType =
|
|||
| typeof AGENT_TYPE_PERMANENT
|
||||
| typeof AGENT_TYPE_TEMPORARY;
|
||||
|
||||
export type AgentStatus = 'offline' | 'error' | 'online' | 'inactive' | 'warning' | 'unenrolling';
|
||||
export type AgentStatus =
|
||||
| 'offline'
|
||||
| 'error'
|
||||
| 'online'
|
||||
| 'inactive'
|
||||
| 'warning'
|
||||
| 'enrolling'
|
||||
| 'unenrolling'
|
||||
| 'degraded';
|
||||
|
||||
export type AgentActionType = 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE' | 'UNENROLL';
|
||||
export interface NewAgentAction {
|
||||
type: AgentActionType;
|
||||
|
@ -82,6 +91,7 @@ interface AgentBase {
|
|||
config_id?: string;
|
||||
config_revision?: number | null;
|
||||
last_checkin?: string;
|
||||
last_checkin_status?: 'error' | 'online' | 'degraded';
|
||||
user_provided_metadata: AgentMetadata;
|
||||
local_metadata: AgentMetadata;
|
||||
}
|
||||
|
|
|
@ -47,6 +47,7 @@ export interface PostAgentCheckinRequest {
|
|||
agentId: string;
|
||||
};
|
||||
body: {
|
||||
status?: 'online' | 'error' | 'degraded';
|
||||
local_metadata?: Record<string, any>;
|
||||
events?: NewAgentEvent[];
|
||||
};
|
||||
|
|
|
@ -178,11 +178,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
|
|||
}
|
||||
|
||||
if (selectedStatus.length) {
|
||||
if (kuery) {
|
||||
kuery = `(${kuery}) and`;
|
||||
}
|
||||
|
||||
kuery = selectedStatus
|
||||
const kueryStatus = selectedStatus
|
||||
.map((status) => {
|
||||
switch (status) {
|
||||
case 'online':
|
||||
|
@ -196,6 +192,12 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
|
|||
return '';
|
||||
})
|
||||
.join(' or ');
|
||||
|
||||
if (kuery) {
|
||||
kuery = `(${kuery}) and ${kueryStatus}`;
|
||||
} else {
|
||||
kuery = kueryStatus;
|
||||
}
|
||||
}
|
||||
|
||||
const agentsRequest = useGetAgents(
|
||||
|
|
|
@ -53,6 +53,22 @@ const Status = {
|
|||
/>
|
||||
</EuiHealth>
|
||||
),
|
||||
Degraded: (
|
||||
<EuiHealth color="danger">
|
||||
<FormattedMessage
|
||||
id="xpack.ingestManager.agentHealth.degradedStatusText"
|
||||
defaultMessage="Degraded"
|
||||
/>
|
||||
</EuiHealth>
|
||||
),
|
||||
Enrolling: (
|
||||
<EuiHealth color="warning">
|
||||
<FormattedMessage
|
||||
id="xpack.ingestManager.agentHealth.enrollingStatusText"
|
||||
defaultMessage="Enrolling"
|
||||
/>
|
||||
</EuiHealth>
|
||||
),
|
||||
Unenrolling: (
|
||||
<EuiHealth color="warning">
|
||||
<FormattedMessage
|
||||
|
@ -67,6 +83,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
|
|||
switch (agent.status) {
|
||||
case 'error':
|
||||
return Status.Error;
|
||||
case 'degraded':
|
||||
return Status.Degraded;
|
||||
case 'inactive':
|
||||
return Status.Inactive;
|
||||
case 'offline':
|
||||
|
@ -75,6 +93,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
|
|||
return Status.Warning;
|
||||
case 'unenrolling':
|
||||
return Status.Unenrolling;
|
||||
case 'enrolling':
|
||||
return Status.Enrolling;
|
||||
default:
|
||||
return Status.Online;
|
||||
}
|
||||
|
|
|
@ -178,8 +178,11 @@ export const postAgentCheckinHandler: RequestHandler<
|
|||
const { actions } = await AgentService.agentCheckin(
|
||||
soClient,
|
||||
agent,
|
||||
request.body.events || [],
|
||||
request.body.local_metadata,
|
||||
{
|
||||
events: request.body.events || [],
|
||||
localMetadata: request.body.local_metadata,
|
||||
status: request.body.status,
|
||||
},
|
||||
{ signal }
|
||||
);
|
||||
const body: PostAgentCheckinResponse = {
|
||||
|
|
|
@ -63,6 +63,7 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
|
|||
config_id: { type: 'keyword' },
|
||||
last_updated: { type: 'date' },
|
||||
last_checkin: { type: 'date' },
|
||||
last_checkin_status: { type: 'keyword' },
|
||||
config_revision: { type: 'integer' },
|
||||
default_api_key_id: { type: 'keyword' },
|
||||
default_api_key: { type: 'binary', index: false },
|
||||
|
@ -310,6 +311,7 @@ export function registerEncryptedSavedObjects(
|
|||
'config_id',
|
||||
'last_updated',
|
||||
'last_checkin',
|
||||
'last_checkin_status',
|
||||
'config_revision',
|
||||
'config_newest_revision',
|
||||
'updated_at',
|
||||
|
|
|
@ -11,7 +11,6 @@ import {
|
|||
AgentEvent,
|
||||
AgentSOAttributes,
|
||||
AgentEventSOAttributes,
|
||||
AgentMetadata,
|
||||
} from '../../../types';
|
||||
|
||||
import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants';
|
||||
|
@ -21,20 +20,24 @@ import { getAgentActionsForCheckin } from '../actions';
|
|||
export async function agentCheckin(
|
||||
soClient: SavedObjectsClientContract,
|
||||
agent: Agent,
|
||||
events: NewAgentEvent[],
|
||||
localMetadata?: any,
|
||||
data: {
|
||||
events: NewAgentEvent[];
|
||||
localMetadata?: any;
|
||||
status?: 'online' | 'error' | 'degraded';
|
||||
},
|
||||
options?: { signal: AbortSignal }
|
||||
) {
|
||||
const updateData: {
|
||||
local_metadata?: AgentMetadata;
|
||||
current_error_events?: string;
|
||||
} = {};
|
||||
const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, events);
|
||||
const updateData: Partial<AgentSOAttributes> = {};
|
||||
const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, data.events);
|
||||
if (updatedErrorEvents) {
|
||||
updateData.current_error_events = JSON.stringify(updatedErrorEvents);
|
||||
}
|
||||
if (localMetadata) {
|
||||
updateData.local_metadata = localMetadata;
|
||||
if (data.localMetadata) {
|
||||
updateData.local_metadata = data.localMetadata;
|
||||
}
|
||||
|
||||
if (data.status !== agent.last_checkin_status) {
|
||||
updateData.last_checkin_status = data.status;
|
||||
}
|
||||
if (Object.keys(updateData).length > 0) {
|
||||
await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData);
|
||||
|
|
|
@ -59,7 +59,7 @@ export function agentCheckinStateConnectedAgentsFactory() {
|
|||
const internalSOClient = getInternalUserSOClient();
|
||||
const now = new Date().toISOString();
|
||||
const updates: Array<SavedObjectsBulkUpdateObject<AgentSOAttributes>> = [
|
||||
...connectedAgentsIds.values(),
|
||||
...agentToUpdate.values(),
|
||||
].map((agentId) => ({
|
||||
type: AGENT_SAVED_OBJECT_TYPE,
|
||||
id: agentId,
|
||||
|
|
|
@ -33,6 +33,7 @@ describe('Agent status service', () => {
|
|||
type: AGENT_TYPE_PERMANENT,
|
||||
attributes: {
|
||||
active: true,
|
||||
last_checkin: new Date().toISOString(),
|
||||
local_metadata: {},
|
||||
user_provided_metadata: {},
|
||||
},
|
||||
|
@ -40,4 +41,36 @@ describe('Agent status service', () => {
|
|||
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||
expect(status).toEqual('online');
|
||||
});
|
||||
|
||||
it('should return enrolling when agent is active but never checkin', async () => {
|
||||
const mockSavedObjectsClient = savedObjectsClientMock.create();
|
||||
mockSavedObjectsClient.get = jest.fn().mockReturnValue({
|
||||
id: 'id',
|
||||
type: AGENT_TYPE_PERMANENT,
|
||||
attributes: {
|
||||
active: true,
|
||||
local_metadata: {},
|
||||
user_provided_metadata: {},
|
||||
},
|
||||
} as SavedObject<AgentSOAttributes>);
|
||||
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||
expect(status).toEqual('enrolling');
|
||||
});
|
||||
|
||||
it('should return unenrolling when agent is unenrolling', async () => {
|
||||
const mockSavedObjectsClient = savedObjectsClientMock.create();
|
||||
mockSavedObjectsClient.get = jest.fn().mockReturnValue({
|
||||
id: 'id',
|
||||
type: AGENT_TYPE_PERMANENT,
|
||||
attributes: {
|
||||
active: true,
|
||||
last_checkin: new Date().toISOString(),
|
||||
unenrollment_started_at: new Date().toISOString(),
|
||||
local_metadata: {},
|
||||
user_provided_metadata: {},
|
||||
},
|
||||
} as SavedObject<AgentSOAttributes>);
|
||||
const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
|
||||
expect(status).toEqual('unenrolling');
|
||||
});
|
||||
});
|
||||
|
|
|
@ -32,6 +32,9 @@ export const PostAgentCheckinRequestSchema = {
|
|||
agentId: schema.string(),
|
||||
}),
|
||||
body: schema.object({
|
||||
status: schema.maybe(
|
||||
schema.oneOf([schema.literal('online'), schema.literal('error'), schema.literal('degraded')])
|
||||
),
|
||||
local_metadata: schema.maybe(schema.recordOf(schema.string(), schema.any())),
|
||||
events: schema.maybe(schema.arrayOf(NewAgentEventSchema)),
|
||||
}),
|
||||
|
|
Loading…
Reference in a new issue