onItemChange(idx, name, value)}
diff --git a/web_console/components/RawDataSelect.jsx b/web_console/components/RawDataSelect.jsx
new file mode 100644
index 000000000..fbb020ec3
--- /dev/null
+++ b/web_console/components/RawDataSelect.jsx
@@ -0,0 +1,33 @@
+import React, { useEffect } from 'react';
+import { Select } from '@zeit-ui/react';
+import useSWR from 'swr';
+import { fetcher } from '../libs/http';
+
+let initialUpdated = false
+
+export default function RawDataSelect(props) {
+ const { data } = useSWR('raw_datas', fetcher);
+ const rawDatas = data ? data.data : [];
+ const actualValue = typeof props.value === 'string'
+ ? props.value : props.value?.name
+ const actualOnChange = (value) => {
+ const rawData = rawDatas.find((x) => x.name === value);
+ props.onChange(rawData);
+ };
+
+ if (!initialUpdated) {
+ typeof props.value === 'string'
+ && rawDatas.length > 0
+ && actualOnChange(props.value)
+
+ initialUpdated = true
+ }
+
+ useEffect(() => () => initialUpdated = false, [])
+
+ return (
+
+ );
+}
diff --git a/web_console/components/ServerTicketSelect.jsx b/web_console/components/ServerTicketSelect.jsx
index 9a3d69b7e..27ede5d63 100644
--- a/web_console/components/ServerTicketSelect.jsx
+++ b/web_console/components/ServerTicketSelect.jsx
@@ -2,23 +2,58 @@ import React from 'react';
import { Select, Popover, Code } from '@zeit-ui/react';
import useSWR from 'swr';
import { fetcher } from '../libs/http';
+import { JOB_TYPE_CLASS } from '../constants/job'
+
+// const mockRes = {
+// "data": [
+// {
+// "name": "ticket-360-k4",
+// "job_type": "data_join",
+// "role": "Follower",
+// "sdk_version": "049ad50",
+// "expire_time": "Fri Jun 18 2021 00:00:00 GMT+0000 (Coordinated Universal Time)",
+// "remark": "",
+// "public_params": "null"
+// },
+// {
+// "name": "test-training",
+// "job_type": "nn_model",
+// "role": "Follower",
+// "sdk_version": "049ad50",
+// "expire_time": "Fri Jun 18 2021 00:00:00 GMT+0000 (Coordinated Universal Time)",
+// "remark": "",
+// "public_params": "null"
+// }
+// ]
+// }
+
+let filter = () => true
+export default function ServerTicketSelect({type, ...props}) {
+
+ if (type) {
+ filter = el => JOB_TYPE_CLASS[type].some(t => el.job_type === t)
+ }
-export default function ServerTicketSelect(props) {
const { data } = useSWR(
props.federation_id ? `federations/${props.federation_id}/tickets` : null,
fetcher,
);
- const tickets = (data && data.data) || [];
- const actualValue = tickets.find((x) => x.name === props.value)?.value;
+ const tickets = data?.data?.filter(filter) || []
+ // const tickets = mockRes.data;
+
+ const actualValue = tickets.find((x) => x.name === props.value)?.name;
const actualOnChange = (value) => {
const ticket = tickets.find((x) => x.name === value);
props.onChange(ticket.name);
};
const popoverContent = (content) => {
+ if (typeof content.public_params === 'string') {
+ content.public_params = JSON.parse(content.public_params || '{}')
+ }
return (
- {JSON.stringify(JSON.parse(content || '{}'), null, 2)}
+ {JSON.stringify(content, null, 2)}
@@ -74,11 +115,10 @@ function FederationItem({ data, onEdit }) {
}
const K8S_SETTINGS_FIELDS = [
- { key: 'namespace', default: K8S_SETTINGS.namespace },
- { key: 'storage_root_path', default: K8S_SETTINGS.storage_root_path },
+ { key: 'namespace' },
+ { key: 'storage_root_path' },
{
key: 'imagePullSecrets',
- default: K8S_SETTINGS.imagePullSecrets,
type: 'json',
span: 24,
path: 'global_replica_spec.template.spec.imagePullSecrets',
@@ -88,7 +128,6 @@ const K8S_SETTINGS_FIELDS = [
key: 'env',
type: 'name-value',
span: 24,
- default: K8S_SETTINGS.env,
props: {
minHeight: '150px',
},
@@ -99,7 +138,6 @@ const K8S_SETTINGS_FIELDS = [
key: 'grpc_spec',
type: 'json',
span: 24,
- default: K8S_SETTINGS.grpc_spec,
props: {
minHeight: '150px',
},
@@ -109,7 +147,6 @@ const K8S_SETTINGS_FIELDS = [
key: 'leader_peer_spec',
type: 'json',
span: 24,
- default: K8S_SETTINGS.leader_peer_spec,
props: {
minHeight: '150px',
},
@@ -119,7 +156,6 @@ const K8S_SETTINGS_FIELDS = [
key: 'follower_peer_spec',
type: 'json',
span: 24,
- default: K8S_SETTINGS.follower_peer_spec,
props: {
minHeight: '150px',
},
@@ -132,7 +168,7 @@ const K8S_SETTINGS_FIELDS = [
*/
function fillField(data, field, edit=false) {
let v = getValueFromJson(data, field.path || field.key)
- if (typeof v === 'object') {
+ if (typeof v === 'object' && v !== null) {
v = JSON.stringify(v, null, 2)
}
field.value = v
@@ -228,7 +264,7 @@ export default function FederationList() {
if (!draft.k8s_settings) { draft.k8s_settings = {} }
for (let field of K8S_SETTINGS_FIELDS) {
fillJSON(
- draft.k8s_settings, field.path || [field.key],
+ draft.k8s_settings, field.path || field.key,
getParsedValueFromData(data.k8s_settings, field)
)
}
@@ -259,10 +295,10 @@ export default function FederationList() {
}
return { newFields }
}
- const DEFAULT_FIELDS = [
+ const DEFAULT_FIELDS = useMemo(() => [
{ key: 'name', required: true },
- { key: 'x-federation' },
{ key: 'trademark' },
+ { key: 'x-federation' },
{ key: 'email' },
{ key: 'tel', label: 'telephone' },
{ key: 'avatar' },
@@ -285,9 +321,14 @@ export default function FederationList() {
]
},
},
- ];
+ ], []);
const [fields, setFields] = useState(DEFAULT_FIELDS);
+ const onClickCreate = () => {
+ setFormMeta({ k8s_settings: K8S_SETTINGS })
+ setFields(mapValueToFields({federation: mapFormMeta2Form(), fields}))
+ setFormVisible(true)
+ }
const toggleForm = () => {
setFormVisible(!formVisible);
if (formVisible) {
@@ -333,10 +374,12 @@ export default function FederationList() {
}
const handleSubmit = (value, groupFormType) => {
- for (let field of K8S_SETTINGS_FIELDS) {
- let error = checkK8sSetting(field, value.k8s_settings[field.key])
- if (error) {
- return {error}
+ if (groupFormType.k8s_settings === 'form') {
+ for (let field of K8S_SETTINGS_FIELDS) {
+ let error = checkK8sSetting(field, value.k8s_settings[field.key])
+ if (error) {
+ return {error}
+ }
}
}
@@ -367,7 +410,7 @@ export default function FederationList() {
<>
Federations
-
+
{federations && federations.map((x) => (
diff --git a/web_console/pages/job/charts/[id].jsx b/web_console/pages/charts/[id].jsx
similarity index 88%
rename from web_console/pages/job/charts/[id].jsx
rename to web_console/pages/charts/[id].jsx
index 1813ea005..4ff807f77 100644
--- a/web_console/pages/job/charts/[id].jsx
+++ b/web_console/pages/charts/[id].jsx
@@ -3,9 +3,9 @@ import useSWR from 'swr';
import { useRouter } from 'next/router';
import css from 'styled-jsx/css';
import { Text, Card, Grid } from '@zeit-ui/react';
-import { fetcher } from '../../../libs/http';
-import Layout from '../../../components/Layout';
-import getJobDashboardUrls from '../../../utils/kibana';
+import { fetcher } from '../../libs/http';
+import Layout from '../../components/Layout';
+import getJobDashboardUrls from '../../utils/kibana';
function useStyle() {
return css`
diff --git a/web_console/pages/datasource/job/[id].jsx b/web_console/pages/datasource/job/[id].jsx
new file mode 100644
index 000000000..c6c67993b
--- /dev/null
+++ b/web_console/pages/datasource/job/[id].jsx
@@ -0,0 +1,9 @@
+import React from 'react';
+import { useRouter } from 'next/router';
+
+import Job from '../../../components/Job';
+
+export default function Jobs() {
+ const router = useRouter();
+ return
+}
diff --git a/web_console/pages/datasource/job/index.jsx b/web_console/pages/datasource/job/index.jsx
new file mode 100644
index 000000000..7a038ee26
--- /dev/null
+++ b/web_console/pages/datasource/job/index.jsx
@@ -0,0 +1,5 @@
+import CommonJobList from '../../../components/CommonJobList'
+
+export default function JobList() {
+ return
+}
\ No newline at end of file
diff --git a/web_console/pages/datasource/tickets/index.jsx b/web_console/pages/datasource/tickets/index.jsx
new file mode 100644
index 000000000..5e8be8d9c
--- /dev/null
+++ b/web_console/pages/datasource/tickets/index.jsx
@@ -0,0 +1,5 @@
+import CommonTicket from '../../../components/CommonTicket'
+
+export default function Ticket() {
+ return
+}
\ No newline at end of file
diff --git a/web_console/pages/index.jsx b/web_console/pages/index.jsx
index fc495aab8..e6e2c0a07 100644
--- a/web_console/pages/index.jsx
+++ b/web_console/pages/index.jsx
@@ -69,7 +69,7 @@ export default function Overview() {
const router = useRouter();
const { data } = useSWR('jobs', fetcher);
const jobs = data ? data.data.filter((x) => x.metadata) : [];
- const goToJob = () => router.push('/job');
+ const goToJob = () => router.push('/datasource/job');
return (
diff --git a/web_console/pages/job/[id].jsx b/web_console/pages/job/[id].jsx
index c77c0f79f..a3bbc1d8a 100644
--- a/web_console/pages/job/[id].jsx
+++ b/web_console/pages/job/[id].jsx
@@ -13,7 +13,7 @@ export default function Job() {
const job = jobData ? jobData.data : null;
return (
-
+ job ?
+ : null
);
}
diff --git a/web_console/pages/raw_data/[id].jsx b/web_console/pages/raw_data/[id].jsx
index 11df26a74..c0203930e 100644
--- a/web_console/pages/raw_data/[id].jsx
+++ b/web_console/pages/raw_data/[id].jsx
@@ -43,7 +43,7 @@ export default function RawDataJob() {
}, [rawData?.localdata?.id]);
return (
-
+ rawData ?
Delete Job
+ : null
);
}
diff --git a/web_console/pages/raw_data/index.jsx b/web_console/pages/raw_data/index.jsx
index 0f701a78d..0124f35cf 100644
--- a/web_console/pages/raw_data/index.jsx
+++ b/web_console/pages/raw_data/index.jsx
@@ -1,5 +1,5 @@
-import React, { useState } from 'react';
-import { Table, Button, Card, Text, Link } from '@zeit-ui/react';
+import React, { useState, useMemo } from 'react';
+import { Table, Button, Card, Text, Link, useTheme } from '@zeit-ui/react';
import NextLink from 'next/link';
import useSWR from 'swr';
import produce from 'immer'
@@ -16,18 +16,19 @@ const RESOURCE_PATH_PREFIX = 'yaml_spec.spec.flReplicaSpecs.[replicaType].templa
const IMAGE_PATH = 'yaml_spec.spec.flReplicaSpecs.[replicaType].template.spec.containers[].image'
const WORKER_REPLICAS_PATH = 'yaml_spec.spec.flReplicaSpecs.Worker.replicas'
+const REPLICA_TYPES = ['Master', 'Worker']
+
const DATA_FORMAT_OPTIONS = [
{ label: 'TF_RECORD', value: 'TF_RECORD' },
{ label: 'CSV_DICT', value: 'CSV_DICT' },
]
const CONTEXT_FIELDS = [
- { key: 'file_wildcard', default: '*', default: RAW_DATA_CONTEXT.file_wildcard },
+ { key: 'file_wildcard' },
{
key: 'input_data_format',
type: 'select',
required: true,
- default: RAW_DATA_CONTEXT.input_data_format,
props: {
options: DATA_FORMAT_OPTIONS
}
@@ -36,7 +37,6 @@ const CONTEXT_FIELDS = [
key: 'output_data_format',
type: 'select',
required: true,
- default: RAW_DATA_CONTEXT.output_data_format,
props: {
options: DATA_FORMAT_OPTIONS
}
@@ -45,7 +45,6 @@ const CONTEXT_FIELDS = [
key: 'compressed_type',
type: 'select',
required: true,
- default: RAW_DATA_CONTEXT.compressed_type,
props: {
options: [
{ label: 'GZIP', value: 'GZIP' },
@@ -54,36 +53,32 @@ const CONTEXT_FIELDS = [
]
}
},
- { key: 'batch_size', default: RAW_DATA_CONTEXT.batch_size },
- { key: 'max_flying_item', default: RAW_DATA_CONTEXT.max_flying_item },
- { key: 'write_buffer_size', default: RAW_DATA_CONTEXT.write_buffer_size },
+ { key: 'batch_size' },
+ { key: 'max_flying_item' },
+ { key: 'write_buffer_size' },
{ key: 'Master Resources', type: 'label', span: 24},
{
key: 'resource.Master.cpu_request',
label: 'cpu request',
path: RESOURCE_PATH_PREFIX + '.requests.cpu',
- default: RAW_DATA_CONTEXT.resource_master_cpu_request,
span: 12,
},
{
key: 'resource.Master.cpu_limit',
label: 'cpu limit',
path: RESOURCE_PATH_PREFIX + '.limits.cpu',
- default: RAW_DATA_CONTEXT.resource_master_cpu_limit,
span: 12
},
{
key: 'resource.Master.memory_request',
label: 'memory request',
path: RESOURCE_PATH_PREFIX + '.requests.memory',
- default: RAW_DATA_CONTEXT.resource_master_memory_request,
span: 12,
},
{
key: 'resource.Master.memory_limit',
label: 'memory limit',
path: RESOURCE_PATH_PREFIX + '.limits.memory',
- default: RAW_DATA_CONTEXT.resource_master_memory_limit,
span: 12
},
{ key: 'worker Resources', type: 'label', span: 24 },
@@ -91,39 +86,37 @@ const CONTEXT_FIELDS = [
key: 'resource.Worker.cpu_request',
label: 'cpu request',
path: RESOURCE_PATH_PREFIX + '.limits.cpu',
- default: RAW_DATA_CONTEXT.resource_master_cpu_request,
span: 12
},
{
key: 'resource.Worker.cpu_limit',
label: 'cpu limit',
path: RESOURCE_PATH_PREFIX + '.limits.cpu',
- default: RAW_DATA_CONTEXT.resource_master_cpu_limit,
span: 12
},
{
key: 'resource.Worker.memory_request',
label: 'memory request',
path: RESOURCE_PATH_PREFIX + '.requests.memory',
- default: RAW_DATA_CONTEXT.resource_master_memory_request,
span: 12,
},
{
key: 'resource.Worker.memory_limit',
label: 'memory limit',
path: RESOURCE_PATH_PREFIX + '.limits.memory',
- default: RAW_DATA_CONTEXT.resource_master_memory_limit,
span: 12
},
{
key: 'num_workers',
label: 'num workers',
span: 12,
- default: 4,
path: WORKER_REPLICAS_PATH
},
]
+/**
+ * write context data to form meta
+ */
function handleContextData(container, data, field) {
if (field.type === 'label') { return }
@@ -140,7 +133,7 @@ function handleContextData(container, data, field) {
}
else if (field.key === 'num_workers') {
- value = parseInt(value || field.default)
+ value = parseInt(value)
}
fillJSON(container, path, value)
@@ -156,16 +149,23 @@ function fillField(data, field) {
const [, replicaType,] = field.key.split('.')
v = getValueFromJson(data, field.path.replace('[replicaType]', replicaType))
}
-
else if (field.key === 'compressed_type') {
v = v === '' ? 'None' : data.compressed_type
}
+ else if (field.key === 'image') {
+ for (let replicaType of REPLICA_TYPES) {
+ v = getValueFromJson(data.context, IMAGE_PATH.replace('[replicaType]', replicaType))
+ if (v) break
+ }
+ }
- if (typeof v === 'object') {
+ if (typeof v === 'object' && v !== null) {
v = JSON.stringify(v, null, 2)
}
- field.value = v
+ if (v) {
+ field.value = v
+ }
field.editing = true
return field
@@ -187,6 +187,9 @@ let formMeta = {}
const setFormMeta = value => { formMeta = value }
export default function RawDataList() {
+
+ const theme = useTheme()
+
const { data, mutate } = useSWR('raw_datas', fetcher);
const rawDatas = data ? data.data : null;
const columns = [
@@ -196,12 +199,12 @@ export default function RawDataList() {
// form meta convert functions
const rewriteFields = (draft, data) => {
// image
- ['Master', 'Worker'].forEach(replicaType => {
+ REPLICA_TYPES.forEach(replicaType => {
fillJSON(draft.context, IMAGE_PATH.replace('[replicaType]', replicaType), data['image'])
})
// output_partition_num
- // data['output_partition_num'] &&
- // fillJSON(draft.context, WORKER_REPLICAS_PATH, data['output_partition_num'])
+ data['output_partition_num'] &&
+ fillJSON(draft.context, WORKER_REPLICAS_PATH, parseInt(data['output_partition_num']))
}
const mapFormMeta2Json = () => {
let data = {}
@@ -274,7 +277,7 @@ export default function RawDataList() {
return { newFields }
}
- const DEFAULT_FIELDS = [
+ const DEFAULT_FIELDS = useMemo(() => [
{ key: 'name', required: true },
{ key: 'federation_id', type: 'federation', label: 'federation', required: true },
{ key: 'output_partition_num', required: true, default: 4 },
@@ -303,9 +306,23 @@ export default function RawDataList() {
]
}
},
- ];
+ ], []);
const [fields, setFields] = useState(DEFAULT_FIELDS)
+ const handleClone = data => {
+ data.context = JSON.parse(data.context)
+
+ setFormMeta(data)
+
+ setFields(fields => mapValueToFields({
+ data: mapFormMeta2Form(fields),
+ fields,
+ type: 'form',
+ }))
+
+ toggleForm()
+ }
+
// eslint-disable-next-line arrow-body-style
const operation = (actions, rowData) => {
// const onConfirm = () => revokeRawData(rowData.rowValue.id);
@@ -314,10 +331,18 @@ export default function RawDataList() {
// };
return (
<>
+ handleClone(rowData.rowValue)}
+ type="success"
+ style={{marginRight: `${theme.layout.gapHalf}`}}
+ >
+ Clone
+
- View Detail
+ Detail
{/* { }} onOk={() => { }}>
Revoke
@@ -338,6 +363,17 @@ export default function RawDataList() {
: [];
const [formVisible, setFormVisible] = useState(false);
+
+ const onCreate = () => {
+ setFormMeta({context: RAW_DATA_CONTEXT})
+ setFields(mapValueToFields({
+ data: mapFormMeta2Form(),
+ fields
+ }))
+
+ toggleForm()
+ }
+
const toggleForm = () => setFormVisible(!formVisible);
const onOk = (rawData) => {
mutate({
@@ -369,7 +405,7 @@ export default function RawDataList() {
<>
RawDatas
-
+
{rawDatas && (
diff --git a/web_console/pages/training/job/[id].jsx b/web_console/pages/training/job/[id].jsx
new file mode 100644
index 000000000..c6c67993b
--- /dev/null
+++ b/web_console/pages/training/job/[id].jsx
@@ -0,0 +1,9 @@
+import React from 'react';
+import { useRouter } from 'next/router';
+
+import Job from '../../../components/Job';
+
+export default function Jobs() {
+ const router = useRouter();
+ return
+}
diff --git a/web_console/pages/training/job/index.jsx b/web_console/pages/training/job/index.jsx
new file mode 100644
index 000000000..48498132f
--- /dev/null
+++ b/web_console/pages/training/job/index.jsx
@@ -0,0 +1,5 @@
+import CommonJobList from '../../../components/CommonJobList'
+
+export default function JobList_() {
+ return
+}
\ No newline at end of file
diff --git a/web_console/pages/training/tickets/index.jsx b/web_console/pages/training/tickets/index.jsx
new file mode 100644
index 000000000..4ee8b4dd8
--- /dev/null
+++ b/web_console/pages/training/tickets/index.jsx
@@ -0,0 +1,5 @@
+import CommonTicket from '../../../components/CommonTicket'
+
+export default function Ticket() {
+ return
+}
\ No newline at end of file
diff --git a/web_console/rpc/client.js b/web_console/rpc/client.js
index 592e8661e..a637cfcaa 100644
--- a/web_console/rpc/client.js
+++ b/web_console/rpc/client.js
@@ -62,6 +62,14 @@ class FederationClient {
deleteJob(params) {
return this._request('deleteJob', params);
}
+
+ updateJob(params) {
+ return this._request('updateJob', params);
+ }
+
+ heartBeat() {
+ return this._request('heartBeat', {});
+ }
}
module.exports = FederationClient;
diff --git a/web_console/rpc/meta.proto b/web_console/rpc/meta.proto
index a906816fa..2e4e81966 100644
--- a/web_console/rpc/meta.proto
+++ b/web_console/rpc/meta.proto
@@ -53,6 +53,27 @@ message DeleteJobResponse {
string message = 1;
}
+message UpdateJobRequest {
+ string name = 1;
+ string job_type = 2;
+ string client_ticket_name = 3;
+ string server_ticket_name = 4;
+ string server_params = 5;
+ string status = 6;
+}
+
+message UpdateJobResponse {
+ Job data = 1;
+ string status = 2;
+}
+
+message HeartBeatRequest {
+}
+
+message HeartBeatResponse {
+ string status = 1;
+}
+
service Federation {
// Obtain available tickets
rpc GetTickets(GetTicketsRequest) returns (GetTicketsResponse) {}
@@ -62,4 +83,10 @@ service Federation {
// Obtain token from preset cipher
rpc DeleteJob(DeleteJobRequest) returns (DeleteJobResponse) {}
+
+ // Obtain token from preset cipher
+ rpc UpdateJob(UpdateJobRequest) returns (UpdateJobResponse) {}
+
+ // Obtain token from preset cipher
+ rpc HeartBeat(HeartBeatRequest) returns (HeartBeatResponse) {}
}
diff --git a/web_console/rpc/server.js b/web_console/rpc/server.js
index 2e2b38107..b7321fd06 100644
--- a/web_console/rpc/server.js
+++ b/web_console/rpc/server.js
@@ -88,7 +88,7 @@ async function createJob(call, callback) {
job_type,
client_ticket_name: server_ticket_name,
server_ticket_name: client_ticket_name,
- server_params,
+ server_params: client_params,
} = call.request;
const ticketRecord = await Ticket.findOne({
where: {
@@ -96,7 +96,9 @@ async function createJob(call, callback) {
},
});
if (!ticketRecord) throw new Error('Ticket not found');
- const params = JSON.parse(server_params);
+ if (ticketRecord.federation_id != federation.id) throw new Error("Invalid ticket name");
+
+ const params = JSON.parse(client_params);
validateTicket(ticketRecord, params);
const [data, created] = await Job.findOrCreate({
@@ -109,7 +111,9 @@ async function createJob(call, callback) {
job_type,
client_ticket_name,
server_ticket_name,
- server_params: JSON.parse(server_params),
+ client_params: JSON.parse(client_params),
+ status: 'started',
+ federation_id: federation.id,
},
});
if (!created) throw new Error('Job already exists');
@@ -123,7 +127,9 @@ async function createJob(call, callback) {
job_type: data.job_type,
client_ticket_name: data.server_ticket_name,
server_ticket_name: data.client_ticket_name,
- server_params: JSON.stringify(data.server_params),
+ server_params: client_params,
+ status: 'started',
+ federation_id: federation.id,
},
});
} catch (err) {
@@ -143,7 +149,9 @@ async function deleteJob(call, callback) {
},
});
if (!job) throw new Error('Job not found');
- await k8s.deleteFLApp(NAMESPACE, job.name);
+ if (!job.status || job.status == 'started') {
+ await k8s.deleteFLApp(NAMESPACE, job.name);
+ }
await job.destroy({ force: true });
callback(null, { message: 'Delete job successfully' });
} catch (err) {
@@ -151,12 +159,111 @@ async function deleteJob(call, callback) {
}
}
+async function updateJob(call, callback) {
+ try {
+ const federation = await authenticate(call.metadata);
+
+ const {
+ name,
+ job_type,
+ client_ticket_name: server_ticket_name,
+ server_ticket_name: client_ticket_name,
+ server_params: client_params,
+ status,
+ } = call.request;
+
+ const new_job = {
+ name, job_type, client_ticket_name, server_ticket_name,
+ client_params: JSON.parse(client_params), status,
+ }
+
+ const old_job = await Job.findOne({
+ where: {
+ name: { [Op.eq]: name },
+ },
+ });
+ if (!old_job) throw new Error(`Job ${name} not found`);
+
+ if (old_job.status === 'error') {
+ throw new Error("Cannot update errored job");
+ }
+ if (old_job.status === 'started' && new_job.status != 'stopped') {
+ throw new Error("Cannot change running job");
+ }
+ if (job_type != old_job.job_type) {
+ throw new Error("Cannot change job type");
+ }
+
+ const ticketRecord = await Ticket.findOne({
+ where: {
+ name: { [Op.eq]: client_ticket_name },
+ },
+ });
+ if (!ticketRecord) throw new Error('Ticket not found');
+ if (ticketRecord.federation_id != federation.id) throw new Error("Invalid ticket name");
+
+ if (ticketRecord.federation_id != old_job.federation_id) {
+ throw new Error("Cannot change job federation");
+ }
+
+ // const params = JSON.parse(client_params);
+ // validateTicket(ticketRecord, params);
+
+ if (old_job.status === 'started' && status === 'stopped') {
+ flapp = (await k8s.getFLApp(NAMESPACE, name)).flapp;
+ pods = (await k8s.getFLAppPods(NAMESPACE, name)).pods;
+ old_job.k8s_meta_snapshot = JSON.stringify({flapp, pods});
+ await k8s.deleteFLApp(NAMESPACE, name);
+ } else if (old_job.status === 'stopped' && new_job.status === 'started') {
+ const args = serverGenerateYaml(federation, new_job, ticketRecord);
+ await k8s.createFLApp(NAMESPACE, args);
+ }
+
+ old_job.client_ticket_name = new_job.client_ticket_name;
+ old_job.server_ticket_name = new_job.server_ticket_name;
+ if (new_job.client_params) {
+ old_job.client_params = new_job.client_params;
+ }
+ old_job.status = new_job.status;
+ old_job.federation_id = new_job.federation_id;
+
+ const data = await old_job.save();
+
+ callback(null, {
+ data: {
+ name: data.name,
+ job_type: data.job_type,
+ client_ticket_name: data.server_ticket_name,
+ server_ticket_name: data.client_ticket_name,
+ server_params: client_params,
+ },
+ status: data.status,
+ });
+ } catch (err) {
+ callback(err);
+ }
+}
+
+/**
+ * get available tickets of current federation
+ */
+async function heartBeat(call, callback) {
+ try {
+ const federation = await authenticate(call.metadata);
+ callback(null, { status: 'success' });
+ } catch (err) {
+ callback(err);
+ }
+}
+
const server = new grpc.Server();
server.addService(pkg.federation.Federation.service, {
getTickets,
createJob,
deleteJob,
+ updateJob,
+ heartBeat,
});
module.exports = server;
diff --git a/web_console/services/index.js b/web_console/services/index.js
index 37d163025..809892442 100644
--- a/web_console/services/index.js
+++ b/web_console/services/index.js
@@ -16,6 +16,10 @@ export async function updateFederation(id, json) {
return client.put(`federations/${id}`, { json }).json();
}
+export async function federationHeartbeat(id) {
+ return client.get(`federations/${id}/heartbeat`).json();
+}
+
export async function createUser(json) {
return client.post('users', { json }).json();
}
diff --git a/web_console/services/job.js b/web_console/services/job.js
index 6b81675d3..913c22c90 100644
--- a/web_console/services/job.js
+++ b/web_console/services/job.js
@@ -7,3 +7,7 @@ export async function deleteJob(name) {
export async function createJob(json) {
return client.post('job', { json }).json();
}
+
+export async function updateJobStatus(id, json) {
+ return client.post(`job/${id}/update`, { json }).json();
+}
\ No newline at end of file
diff --git a/web_console/startup.sh b/web_console/startup.sh
new file mode 100755
index 000000000..70f2279b6
--- /dev/null
+++ b/web_console/startup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+npx sequelize-cli db:migrate \
+ --url="${DB_DIALECT:-mysql}://${DB_USERNAME:-fedlearner}:${DB_PASSWORD:-fedlearner}@${DB_HOST:-127.0.0.1}:${DB_PORT:-3306}/${DB_DATABASE:-fedlearner}"
+
+node bootstrap.js &
+
+while true; do sleep 1000; done
diff --git a/web_console/tests/utils/form_utils.test.js b/web_console/tests/utils/form_utils.test.js
index f4db54f05..7357a89fb 100644
--- a/web_console/tests/utils/form_utils.test.js
+++ b/web_console/tests/utils/form_utils.test.js
@@ -46,4 +46,55 @@ describe('form_utils', () => {
}
})
})
+
+ it('test push new object and insert to the first item', () => {
+ let paths = [
+ 'foo.bar[].baz.qux',
+ 'foo.bar[-1].baz.quux',
+ ]
+ let value = 1
+ let obj = {}
+ paths.forEach(path => fillJSON(obj, path, value))
+ assert.deepStrictEqual(obj, {
+ foo: {
+ bar: [
+ { baz: { qux: value} },
+ { baz: { quux: value} },
+ ]
+ }
+ })
+ })
+
+ it('test multi push', () => {
+ let paths = [
+ 'foo.bar[].baz[].qux',
+ 'foo.bar[].baz[-1].quux',
+ ]
+ let value = 1
+ let obj = {}
+ paths.forEach(path => fillJSON(obj, path, value))
+ assert.deepStrictEqual(obj, {
+ foo: {
+ bar: [
+ { baz: [{ qux: value }, { quux: value } ] },
+ ]
+ }
+ })
+ })
+})
+
+describe('filter value', () => {
+ it('should filter undefined', () => {
+ let arr = [1, 2, undefined, 3, undefined]
+ filterArrayValue(arr)
+
+ assert.deepStrictEqual(arr, [1, 2, 3])
+ })
+
+ it('should filter value', () => {
+ let arr = [1, 2, undefined, 3, undefined]
+ filterArrayValue(arr, 3)
+
+ assert.deepStrictEqual(arr, [1, 2, undefined, undefined])
+ })
})
\ No newline at end of file
diff --git a/web_console/utils/form_utils.js b/web_console/utils/form_utils.js
index d3fb66552..1be3a0219 100644
--- a/web_console/utils/form_utils.js
+++ b/web_console/utils/form_utils.js
@@ -2,38 +2,49 @@
* fill a json object with given path
* path rules:
* - `path.of.some.key` will make obj.path.of.some.key === value
- * - `array[].key` will make obj.array[0].key === value
+ * - `array[].key` will always make obj.array[0].key === value
+ * - `array[-1].key` will push a new object to array
+ * TODO: `array[],[]key`
*/
-export const fillJSON = (container, path, value) => {
- let containerIsArray = Array.isArray(container)
- if (typeof path === 'string') {
- path = path.split('.')
- }
- if (path.length === 1) {
- if (containerIsArray) {
- container[0]
- ? container[0][path[0]]= value
- : (container[0] = { [path[0]]: value })
- } else {
- container[path[0]] = value
+export function fillJSON(container, path, value) {
+ let paths = path.split('.')
+ let currLayer = container
+
+ let cursor = 0
+ while (cursor < paths.length) {
+ let arrayMarks = new RegExp(/\[([\s\S]*?)\]$/).exec(paths[cursor])
+ let isArray = !!arrayMarks
+ let currKey = isArray ? paths[cursor].replace(arrayMarks[0], '') : paths[cursor]
+
+ // insert value
+ if (cursor === paths.length - 1) {
+ currLayer[paths[cursor]] = value
+ break
}
- return
- }
- let currLayer = container
- let currLayerIsArray = path[0].endsWith('[]')
- let currPath = currLayerIsArray ? path[0].replace('[]', '') : path[0]
+ // handle layer
+ if (isArray) {
+ let posToInsert = parseInt(arrayMarks[1] || '0')
- if (containerIsArray) {
- !container[0] && (container[0] = {})
- currLayer = container[0]
- }
+ if (!currLayer[currKey]) { currLayer[currKey] = [] }
- if (currLayer[currPath] === undefined) {
- currLayer[currPath] = currLayer[currPath] || (currLayerIsArray ? [] : {})
- }
+ switch (posToInsert) {
+ case 0:
+ currLayer[currKey][0] = currLayer[currKey][0] || {}
+ currLayer = currLayer[currKey][0]
+ break
+ case -1:
+ let newObj = {}
+ currLayer[currKey] = currLayer[currKey].concat(newObj)
+ currLayer = newObj
+ }
+ } else {
+ if (!currLayer[currKey]) { currLayer[currKey] = {} }
- fillJSON(currLayer[currPath], path.slice(1), value)
+ currLayer = currLayer[currKey]
+ }
+ cursor++
+ }
}
/**
@@ -43,6 +54,7 @@ export const fillJSON = (container, path, value) => {
* - `array[].key` will return obj.array[0].key
*/
export const getValueFromJson = (data, path) => {
+ if (!data) return
if (typeof path === 'string') {
path = path.split('.')
}
@@ -57,9 +69,43 @@ export const getValueFromJson = (data, path) => {
}
export function getParsedValueFromData (data, field) {
- let value = (data && data[field.key]) || (field.emptyDefault || '')
+ let value = (data && data[field.key])
+
if (['json', 'name-value'].some(el => el === field.type)) {
- value = JSON.parse(value || '{}')
+ value = value ? JSON.parse(value) : field.emptyDefault || {}
+ }
+ else if (field.type === 'bool-select') {
+ value = typeof value === 'boolean' ? value : true
}
+ else {
+ value = value || data[field.key] || ''
+ }
+
return value
+}
+
+/**
+ * filter a value from an array.
+ * example: [1, 2, undefined] -> filterArrayValue(arr) -> [1, 2]
+ */
+export function filterArrayValue (arr, value = undefined) {
+ for (let i = arr.length - 1; i >= 0; i--) {
+ if (arr[i] === value) {
+ arr.splice(i, 1)
+ }
+ }
+ return arr
+}
+
+/**
+ * get value of an item of env array
+ */
+export function getValueFromEnv(data, envPath, name) {
+ let envs = getValueFromJson(data, envPath)
+ if (!envs) { envs = [] }
+ let envNames = envs.map(env => env.name)
+ let v = envs[envNames.indexOf(name)]
+ v = v && v.value || ''
+
+ return v
}
\ No newline at end of file
diff --git a/web_console/utils/job.js b/web_console/utils/job.js
index 6e2fc3b56..0ce7d384d 100644
--- a/web_console/utils/job.js
+++ b/web_console/utils/job.js
@@ -5,6 +5,14 @@ export const FLAppStatus = {
ShutDown: 'FLStateShutDown',
};
+export const JobStatus = {
+ Running: 'Running',
+ Failed: 'Failed',
+ Complete: 'Complete',
+ ShutDown: 'ShutDown',
+ Killed: 'Killed',
+}
+
export function handleStatus(statusStr) {
if (typeof statusStr !== 'string') return statusStr;
return statusStr.replace('FLState', '');
@@ -13,14 +21,25 @@ export function handleStatus(statusStr) {
export function getStatusColor(statusStr) {
switch (statusStr) {
case FLAppStatus.Running:
+ case JobStatus.Running:
return 'lightblue';
case FLAppStatus.Failed:
+ case JobStatus.Failed:
return 'red';
case FLAppStatus.Complete:
+ case JobStatus.Complete:
return 'limegreen';
case FLAppStatus.ShutDown:
+ case JobStatus.ShutDown:
return 'brown';
default:
return undefined;
}
}
+
+export function getJobStatus(job) {
+ if (job.localdata?.status === 'stopped') {
+ return JobStatus.Killed
+ }
+ return handleStatus(job?.status?.appState)
+}
\ No newline at end of file
diff --git a/web_console/utils/job_builder.js b/web_console/utils/job_builder.js
index ab64a5347..ecd9c1fbd 100644
--- a/web_console/utils/job_builder.js
+++ b/web_console/utils/job_builder.js
@@ -12,10 +12,17 @@ const assert = require('assert');
const lodash = require('lodash');
const getConfig = require('./get_confg');
-const { NAMESPACE, ES_HOST, ES_PORT } = getConfig({
+const { NAMESPACE, ES_HOST, ES_PORT, DB_HOST, DB_PORT,
+ DB_DATABASE, DB_USERNAME, DB_PASSWORD, KVSTORE_TYPE } = getConfig({
NAMESPACE: process.env.NAMESPACE,
ES_HOST: process.env.ES_HOST,
ES_PORT: process.env.ES_PORT,
+ DB_HOST: process.env.DB_HOST,
+ DB_PORT: process.env.DB_PORT,
+ DB_DATABASE: process.env.DB_DATABASE,
+ DB_USERNAME: process.env.DB_USERNAME,
+ DB_PASSWORD: process.env.DB_PASSWORD,
+ KVSTORE_TYPE: process.env.KVSTORE_TYPE,
});
function joinPath(base, ...rest) {
@@ -68,13 +75,30 @@ function validateTicket(ticket, params) {
}
function clientValidateJob(job, client_ticket, server_ticket) {
+ if (job.job_type != client_ticket.job_type) {
+ throw new Error(`client_ticket.job_type ${client_ticket.job_type} does not match job.job_type ${job.job_type}`);
+ }
+ if (job.job_type != server_ticket.job_type) {
+ throw new Error(`server_ticket.job_type ${server_ticket.job_type} does not match job.job_type ${job.job_type}`);
+ }
+ if (client_ticket.role === server_ticket.role) {
+ throw new Error(`client_ticket.role ${client_ticket.role} must be different from server_ticket.role ${server_ticket.role}`);
+ }
+
+ if (job.server_params) {
+ let client_replicas = job.client_params.spec.flReplicaSpecs["Worker"]["replicas"];
+ let server_replicas = job.server_params.spec.flReplicaSpecs["Worker"]["replicas"];
+ if (client_replicas != server_replicas) {
+ throw new Error(`replicas in client_params ${client_replicas} is different from replicas in server_params ${server_replicas}`);
+ }
+ }
return true;
}
-// Only allow some fields to be used from job.server_params because
+// Only allow some fields to be used from job params because
// it is received from peers and cannot be totally trusted.
function extractPermittedJobParams(job) {
- const params = job.server_params;
+ const params = job.client_params;
const permitted_envs = permittedJobEnvs[job.job_type];
const extracted = {};
@@ -168,6 +192,7 @@ function generateYaml(federation, job, job_params, ticket) {
restartPolicy: 'Never',
containers: [{
env: [
+ { name: 'STORAGE_ROOT_PATH', value: k8s_settings.storage_root_path },
{ name: 'POD_IP', valueFrom: { fieldRef: { fieldPath: 'status.podIP' } } },
{ name: 'POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
{ name: 'ROLE', value: ticket.role.toLowerCase() },
@@ -179,6 +204,12 @@ function generateYaml(federation, job, job_params, ticket) {
{ name: 'MEM_LIMIT', valueFrom: { resourceFieldRef: { resource: 'limits.memory' } } },
{ name: 'ES_HOST', value: ES_HOST },
{ name: 'ES_PORT', value: `${ES_PORT}` },
+ { name: 'DB_HOST', value: `${DB_HOST}` },
+ { name: 'DB_PORT', value: `${DB_PORT}` },
+ { name: 'DB_DATABASE', value: `${DB_DATABASE}` },
+ { name: 'DB_USERNAME', value: `${DB_USERNAME}` },
+ { name: 'DB_PASSWORD', value: `${DB_PASSWORD}` },
+ { name: 'KVSTORE_TYPE', value: `${KVSTORE_TYPE}` },
],
imagePullPolicy: 'IfNotPresent',
name: 'tensorflow',
@@ -252,10 +283,17 @@ function portalGenerateYaml(federation, raw_data) {
restartPolicy: 'Never',
containers: [{
env: [
+ { name: 'STORAGE_ROOT_PATH', value: k8s_settings.storage_root_path },
{ name: 'POD_IP', valueFrom: { fieldRef: { fieldPath: 'status.podIP' } } },
{ name: 'POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
{ name: 'ES_HOST', value: ES_HOST },
{ name: 'ES_PORT', value: `${ES_PORT}` },
+ { name: 'DB_HOST', value: `${DB_HOST}` },
+ { name: 'DB_PORT', value: `${DB_PORT}` },
+ { name: 'DB_DATABASE', value: `${DB_DATABASE}` },
+ { name: 'DB_USERNAME', value: `${DB_USERNAME}` },
+ { name: 'DB_PASSWORD', value: `${DB_PASSWORD}` },
+ { name: 'KVSTORE_TYPE', value: `${KVSTORE_TYPE}` },
{ name: 'APPLICATION_ID', value: raw_data.name },
{ name: 'DATA_PORTAL_NAME', value: raw_data.name },
{ name: 'OUTPUT_PARTITION_NUM', value: `${raw_data.output_partition_num}` },
@@ -281,6 +319,7 @@ function portalGenerateYaml(federation, raw_data) {
restartPolicy: 'Never',
containers: [{
env: [
+ { name: 'STORAGE_ROOT_PATH', value: k8s_settings.storage_root_path },
{ name: 'POD_IP', valueFrom: { fieldRef: { fieldPath: 'status.podIP' } } },
{ name: 'POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
{ name: 'CPU_REQUEST', valueFrom: { resourceFieldRef: { resource: 'requests.cpu' } } },
@@ -289,6 +328,12 @@ function portalGenerateYaml(federation, raw_data) {
{ name: 'MEM_LIMIT', valueFrom: { resourceFieldRef: { resource: 'limits.memory' } } },
{ name: 'ES_HOST', value: ES_HOST },
{ name: 'ES_PORT', value: `${ES_PORT}` },
+ { name: 'DB_HOST', value: `${DB_HOST}` },
+ { name: 'DB_PORT', value: `${DB_PORT}` },
+ { name: 'DB_DATABASE', value: `${DB_DATABASE}` },
+ { name: 'DB_USERNAME', value: `${DB_USERNAME}` },
+ { name: 'DB_PASSWORD', value: `${DB_PASSWORD}` },
+ { name: 'KVSTORE_TYPE', value: `${KVSTORE_TYPE}` },
{ name: 'APPLICATION_ID', value: raw_data.name },
{ name: 'BATCH_SIZE', value: raw_data.context.batch_size ? `${raw_data.context.batch_size}` : "" },
{ name: 'INPUT_DATA_FORMAT', value: raw_data.context.input_data_format },
diff --git a/web_console/utils/kibana.js b/web_console/utils/kibana.js
index d76e4f7ea..f3b0aa249 100644
--- a/web_console/utils/kibana.js
+++ b/web_console/utils/kibana.js
@@ -2,8 +2,160 @@ const dayjs = require('dayjs');
const getConfig = require('./get_confg');
const JOB_METRICS = {
- data_join: [],
- psi_data_join: [],
+ data_join: [
+ {
+ query: "name%20:%22data_block_dump_duration%22",
+ mode: "avg",
+ title: "data_block_dump_duration"
+ },
+ {
+ query: "name%20:%22data_block_index%22",
+ mode: "avg",
+ title: "data_block_index"
+ },
+ {
+ query: "name%20:%22stats_cum_join_num%22",
+ mode: "avg",
+ title: "stats_cum_join_num"
+ },
+ {
+ query: "name%20:%22actual_cum_join_num%22",
+ mode: "avg",
+ title: "actual_cum_join_num"
+ },
+ {
+ query: "name%20:%22leader_stats_index%22",
+ mode: "avg",
+ title: "leader_stats_index"
+ },
+ {
+ query: "name%20:%22follower_stats_index%22",
+ mode: "avg",
+ title: "follower_stats_index"
+ },
+ {
+ query: "name%20:%22example_id_dump_duration%22",
+ mode: "avg",
+ title: "example_id_dump_duration"
+ },
+ {
+ query: "name%20:%22example_dump_file_index%22",
+ mode: "avg",
+ title: "example_dump_file_index"
+ },
+ {
+ query: "name%20:%22example_id_dumped_index%22",
+ mode: "avg",
+ title: "example_id_dumped_index"
+ },
+ {
+ query: "name%20:%22stats_cum_join_num%22",
+ mode: "avg",
+ title: "stats_cum_join_num"
+ },
+ {
+ query: "name%20:%22actual_cum_join_num%22",
+ mode: "avg",
+ title: "actual_cum_join_num"
+ },
+ {
+ query: "name%20:%22leader_stats_index%22",
+ mode: "avg",
+ title: "leader_stats_index"
+ },
+ {
+ query: "name%20:%22follower_stats_index%22",
+ mode: "avg",
+ title: "follower_stats_index"
+ },
+ {
+ query: "name%20:%22leader_join_rate_percent%22",
+ mode: "avg",
+ title: "leader_join_rate_percent"
+ },
+ {
+ query: "name%20:%22follower_join_rate_percent%22",
+ mode: "avg",
+ title: "follower_join_rate_percent"
+ }
+ ],
+ psi_data_join: [
+ {
+ query: "name%20:%22data_block_dump_duration%22",
+ mode: "avg",
+ title: "data_block_dump_duration"
+ },
+ {
+ query: "name%20:%22data_block_index%22",
+ mode: "avg",
+ title: "data_block_index"
+ },
+ {
+ query: "name%20:%22stats_cum_join_num%22",
+ mode: "avg",
+ title: "stats_cum_join_num"
+ },
+ {
+ query: "name%20:%22actual_cum_join_num%22",
+ mode: "avg",
+ title: "actual_cum_join_num"
+ },
+ {
+ query: "name%20:%22leader_stats_index%22",
+ mode: "avg",
+ title: "leader_stats_index"
+ },
+ {
+ query: "name%20:%22follower_stats_index%22",
+ mode: "avg",
+ title: "follower_stats_index"
+ },
+ {
+ query: "name%20:%22example_id_dump_duration%22",
+ mode: "avg",
+ title: "example_id_dump_duration"
+ },
+ {
+ query: "name%20:%22example_dump_file_index%22",
+ mode: "avg",
+ title: "example_dump_file_index"
+ },
+ {
+ query: "name%20:%22example_id_dumped_index%22",
+ mode: "avg",
+ title: "example_id_dumped_index"
+ },
+ {
+ query: "name%20:%22stats_cum_join_num%22",
+ mode: "avg",
+ title: "stats_cum_join_num"
+ },
+ {
+ query: "name%20:%22actual_cum_join_num%22",
+ mode: "avg",
+ title: "actual_cum_join_num"
+ },
+ {
+ query: "name%20:%22leader_stats_index%22",
+ mode: "avg",
+ title: "leader_stats_index"
+ },
+ {
+ query: "name%20:%22follower_stats_index%22",
+ mode: "avg",
+ title: "follower_stats_index"
+ },
+ {
+ query: "name%20:%22leader_join_rate_percent%22",
+ mode: "avg",
+ title: "leader_join_rate_percent"
+ },
+ {
+ query: "name%20:%22follower_join_rate_percent%22",
+ mode: "avg",
+ title: "follower_join_rate_percent"
+ }
+ ],
tree_model: [],
nn_model: [
{