Merge pull request #950 from singnet/data-preset-tab

Dataset processing

package.json

@@ -40,6 +40,7 @@
     "react-slick": "^0.30.2",
     "react-star-rating-component": "^1.4.1",
     "react-swipeable-views": "^0.14.0",
+    "recharts": "^2.13.3",
     "redux": "^5.0.1",
     "redux-thunk": "^3.1.0",
     "singularitynet-platform-contracts": "^1.0.4",

src/Redux/actionCreators/DatasetActions.js

@@ -0,0 +1,100 @@
+import { DatasetClient, DatasetEndpoints } from "../../config/DatasetClient";
+
+export const SET_MAIN_DATASET = "SET_MAIN_DATASET";
+export const SET_MERGE_DATASET = "SET_MERGE_DATASET";
+export const SET_EXAMPLE_DATASETS = "SET_EXAMPLE_DATASETS";
+export const CLEAR_EXAMPLE_DATASETS = "CLEAR_EXAMPLE_DATASETS";
+export const ADD_RECENT_DATASET = "ADD_RECENT_DATASET";
+export const UPDATE_RECENT_DATASET = "UPDATE_RECENT_DATASET";
+export const CLEAR_RECENT_DATASETS = "CLEAR_RECENT_DATASETS";
+
+export const setMainDataset = (dataset) => (dispatch) => {
+  dispatch({
+    type: SET_MAIN_DATASET,
+    payload: dataset,
+  });
+};
+
+export const setMergeDataset = (dataset) => (dispatch) => {
+  dispatch({
+    type: SET_MERGE_DATASET,
+    payload: dataset,
+  });
+};
+
+export const setExampleDatasets = (datasetList) => (dispatch) => {
+  dispatch({
+    type: SET_EXAMPLE_DATASETS,
+    payload: datasetList,
+  });
+};
+
+export const clearExampleDatasets = () => (dispatch) => {
+  dispatch({
+    type: CLEAR_EXAMPLE_DATASETS,
+  });
+};
+
+export const addRecentDataset = (dataset) => (dispatch) => {
+  dispatch({
+    type: ADD_RECENT_DATASET,
+    payload: dataset,
+  });
+};
+
+export const updateRecentDataset = (datasetKey, additionalInfo) => (dispatch) => {
+  dispatch({
+    type: UPDATE_RECENT_DATASET,
+    payload: { datasetKey, additionalInfo },
+  });
+};
+
+export const clearRecentDatasets = () => (dispatch) => {
+  dispatch({
+    type: CLEAR_RECENT_DATASETS,
+  });
+};
+
+export const getDatasetStatistic = (datasetKey) => async (dispatch) => {
+  const params = new URLSearchParams([["dataset_key", datasetKey]]);
+  return DatasetClient.get(DatasetEndpoints.VALIDATE_AND_ANALIZE, { params });
+};
+
+export const improveDataset = (datasetKey, improveOptionsList) => async (dispatch) => {
+  const params = {
+    dataset_key: datasetKey,
+    improve_options: improveOptionsList.reduce((acc, field) => {
+      acc[field] = true;
+      return acc;
+    }, {}),
+  };
+  return DatasetClient.post(DatasetEndpoints.IMPROVE, params);
+};
+
+export const mergeDatasets = (mainDataset, mergeDataset) => async (dispatch) => {
+  const params = {
+    dataset_key_base: mainDataset,
+    dataset_key_additional: mergeDataset,
+  };
+  return DatasetClient.post(DatasetEndpoints.VALIDATE_MERGE, params)
+    .then((response) => {
+      return response.data;
+    })
+    .catch((error) => {
+      console.error("mergeDatasets Error:", error);
+    });
+};
+
+export const validateMergeDatasets = (mainDataset, mergeDataset) => async (dispatch) => {
+  const params = {
+    dataset_key_base: mainDataset,
+    dataset_key_additional: mergeDataset,
+  };
+  return DatasetClient.post(DatasetEndpoints.MERGE, params)
+    .then((response) => {
+      return response.data;
+    })
+    .catch((error) => {
+      console.error("validateMergeDatasets Error:", error);
+    });
+};

src/Redux/actionCreators/ServiceTrainingActions.js

@@ -2,8 +2,9 @@ import axios from "axios";
 import { LoaderContent } from "../../utility/constants/LoaderContent";
 import { startAppLoader, stopAppLoader } from "./LoaderActions";
 import { getServiceClient } from "./SDKActions";
-import { updateMetamaskWallet } from "./UserActions";
+import { fetchAuthenticatedUser, updateMetamaskWallet } from "./UserActions";
 import { modelStatus } from "../reducers/ServiceTrainingReducer";
+// import { userActions } from ".";
 export const SET_MODEL_DETAILS = "SET_MODEL_DETAILS";
 export const SET_MODELS_LIST = "SET_MODELS_LIST";
 export const RESET_MODEL_DETAILS = "RESET_MODEL_DETAILS";
@@ -189,20 +190,49 @@ const modelStatusByNumber = {
   4: "DELETED",
 };
 
-export const publishDatasetToS3 = async (fileBlob, name) => {
+export const publishDatasetForTraining = (fileBlob, name) => async (dispatch) => {
+  const linkAndKeyDataset = await dispatch(
+    publishDatasetToS3(
+      fileBlob,
+      name,
+      "https://xim5yugo7g.execute-api.us-east-1.amazonaws.com/default",
+      "S1kDjcub9k78JFAyrLPsfS0yQoQ4mgmmpeWKlIoVvYsk6JVq5v4HHKvKQgZ0VdI7"
+    )
+  );
+  return linkAndKeyDataset;
+};
+
+export const publishDatasetForImproving = (fileBlob, name) => async (dispatch) => {
+  const linkAndKeyDataset = await dispatch(
+    publishDatasetToS3(
+      fileBlob,
+      name,
+      "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com",
+      "IYE2sz0hUSGhWcyLQTwXS0AbiXKq4h1eW85MZSo6uDhtYfXI8dXisTzRyXaBCImH"
+    )
+  );
+  return linkAndKeyDataset;
+};
+
+export const publishDatasetToS3 = (fileBlob, name, baseUrl, authToken) => async (dispatch) => {
+  const { email } = await dispatch(fetchAuthenticatedUser());
+
   try {
-    const fileKey = Date.now() + "_" + name;
-    const url = `https://xim5yugo7g.execute-api.us-east-1.amazonaws.com/default/upload?key=${fileKey}`;
+    const fileKey = name + "_" + email + "_" + Date.now();
+    const url = `${baseUrl}/upload?key=${fileKey}`;
 
     let instance = axios.create({
       headers: {
-        Authorization: "S1kDjcub9k78JFAyrLPsfS0yQoQ4mgmmpeWKlIoVvYsk6JVq5v4HHKvKQgZ0VdI7",
+        Authorization: authToken,
       },
     });
 
     const response = await instance.get(url);
     await axios.put(response.data.uploadURL, fileBlob);
-    return `https://xim5yugo7g.execute-api.us-east-1.amazonaws.com/default/download?key=${fileKey}`;
+    return {
+      url: `${baseUrl}/download?key=${fileKey}`,
+      datasetKey: fileKey
+    };
   } catch (err) {
     throw new Error(err);
   }

src/Redux/actionCreators/index.js

@@ -8,6 +8,7 @@ import * as stylesActions from "./StylesActions";
 import * as paymentActions from "./PaymentActions";
 import * as uiContentActions from "./UiContentActions";
 import * as sdkActions from "./SDKActions";
+import * as datasetActions from "./DatasetActions";
 
 export {
   sdkActions,
@@ -20,4 +21,5 @@ export {
   stylesActions,
   paymentActions,
   uiContentActions,
+  datasetActions,
 };

src/Redux/reducers/DatasetReducer.js

@@ -0,0 +1,92 @@
+import { datasetActions } from "../actionCreators";
+
+const initialState = {
+  mainDataset: null,
+  mergeDataset: null,
+  exampleDatasets: [
+    {
+      datasetKey: "[email protected]_1733211488305",
+      name: "DataSet 1: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1733211612606",
+      name: "DataSet 2: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1733211645798",
+      name: "DataSet 3: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1733211676614",
+      name: "DataSet 4: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+  ],
+  recentDatasets: [
+    {
+      datasetKey: "[email protected]_1732864358004",
+      // link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
+      name: "DataSet Recent 1: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1732864358004",
+      // link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
+      name: "DataSet Recent 2: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1732864358004",
+      // link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
+      name: "DataSet Recent 3: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+    {
+      datasetKey: "[email protected]_1732864358004",
+      // link: "https://ozx0e68owf.execute-api.us-east-1.amazonaws.com/download?key=data_instruct_llm_1000.zip_training@singularitynet.io_1732864358004",
+      name: "DataSet Recent 4: Training data for text translation",
+      size: 51234,
+      tag: "Text",
+    },
+  ],
+};
+
+const datasetReducer = (state = initialState, action) => {
+  switch (action.type) {
+    case datasetActions.SET_MAIN_DATASET:
+      return { ...state, mainDataset: action.payload };
+    case datasetActions.SET_MERGE_DATASET:
+      return { ...state, mergeDataset: action.payload };
+    case datasetActions.SET_EXAMPLE_DATASETS:
+      return { ...state, exampleDatasets: action.payload };
+    case datasetActions.CLEAR_EXAMPLE_DATASETS:
+      return { ...state, exampleDatasets: [] };
+    case datasetActions.ADD_RECENT_DATASET:
+      return { ...state, recentDatasets: [action.payload, ...state.recentDatasets] };
+    case datasetActions.UPDATE_RECENT_DATASET:
+      return {
+        ...state,
+        recentDatasets: state.recentDatasets.map((dataset) =>
+          dataset.datasetKey === action.payload.datasetKey
+            ? { ...dataset, additionalInfo: action.payload.additionalInfo } // Update the matched user
+            : dataset
+        ),
+      };
+    case datasetActions.CLEAR_RECENT_DATASETS:
+      return { ...state, recentDatasets: [] };
+    default:
+      return state;
+  }
+};
+
+export default datasetReducer;

src/Redux/reducers/index.js

@@ -9,6 +9,7 @@ import stylesReducer from "./StylesReducer";
 import paymentReducer from "./PaymentReducer";
 import uiContentReducer from "./UiContentReducer";
 import sdkReducer from "./SDKReducer";
+import datasetReducer from "./DatasetReducer";
 
 const rootReducer = combineReducers({
   userReducer,
@@ -21,6 +22,7 @@ const rootReducer = combineReducers({
   paymentReducer,
   uiContentReducer,
   sdkReducer,
+  datasetReducer,
 });
 
 export default rootReducer;

src/components/ServiceDetails/AboutService/ServiceOverview.js

@@ -41,7 +41,7 @@ const ServiceOverview = ({ classes, description, tags, isTrainingAvailable }) =>
             <div className={classes.trainingLink}>
               <p>For this service you can create your own training model!</p>
               {/* //TODO */}
-              <Link className={classes.tryTrainingBtn} to={location.pathname.split("tab/")[0] + "tab/" + 2}>
+              <Link className={classes.tryTrainingBtn} to={location.pathname.split("tab/")[0] + "tab/" + 3}>
                 Try now!
               </Link>
             </div>

src/components/ServiceDetails/DataPreset/DashboardModal/ButtonGroup.js

@@ -0,0 +1,62 @@
+import { withStyles } from "@mui/styles";
+import { useStyles } from "./styles";
+import StyledButton from "../../../common/StyledButton";
+import TableIcon from "@mui/icons-material/TableChartOutlined";
+import { useDispatch } from "react-redux";
+import {
+  addRecentDataset,
+  improveDataset,
+  setMainDataset, setMergeDataset,
+} from "../../../../Redux/actionCreators/DatasetActions";
+import { startAppLoader, stopAppLoader } from "../../../../Redux/actionCreators/LoaderActions";
+import { LoaderContent } from "../../../../utility/constants/LoaderContent";
+
+const ButtonsGroup = ({ classes, selectedParameters, isTableView, toggleTableView, dataset, index }) => {
+  const dispatch = useDispatch();
+
+  const tableButtonText = isTableView ? "close tablet" : "view tablet";
+
+  const isImproveButtonDisable = !selectedParameters?.size;
+
+  const getImprovedDataset = async () => {
+    try{
+      dispatch(startAppLoader(LoaderContent.IMPROVE_DATASET));
+      const { data } = await dispatch(improveDataset(dataset.datasetKey, Array.from(selectedParameters.keys())));
+      const improvedDataset = {
+        additionalInfo: {
+          analysis: data.analysis,
+          datasaet_sample: data.dataset_sample
+        },
+        datasetKey: data.dataset_key_new,
+        name: dataset.name + "_improved_rate_" + data.analysis.overall_score,
+        size: dataset.size,
+        tag: dataset.tag,
+      };
+      await dispatch(addRecentDataset(improvedDataset));
+      !index ? await dispatch(setMainDataset(improvedDataset)) : await dispatch(setMergeDataset(improvedDataset));
+    } catch (error) {
+      console.error("getImprovedDataset error", error);
+    } finally {
+      dispatch(stopAppLoader());
+    }
+  };
+
+  return (
+    <div className={classes.buttonsContainer}>
+      <StyledButton
+        type="transparentBlueBorder"
+        btnText={tableButtonText}
+        IconComponent={TableIcon}
+        onClick={toggleTableView}
+      />
+      <StyledButton
+        disabled={isImproveButtonDisable}
+        type="gradientAccent"
+        btnText="IMPROVE"
+        onClick={getImprovedDataset}
+      />
+    </div>
+  );
+};
+
+export default withStyles(useStyles)(ButtonsGroup);