孙浩博，fixed：单文件数据集上传功能

pom.xml

@@ -291,6 +291,11 @@
 			<artifactId>pinyin4j</artifactId>
 			<version>2.5.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.dom4j</groupId>
+			<artifactId>dom4j</artifactId>
+			<version>2.1.4</version>
+		</dependency>
 		<!--<dependency>-->
 			<!--<groupId>jdk.tools</groupId>-->
 			<!--<artifactId>jdk.tools</artifactId>-->

src/main/java/io/renren/common/utils/DockerClientUtils.java

@@ -7,6 +7,7 @@ import com.spotify.docker.client.LogStream;
 import com.spotify.docker.client.exceptions.DockerCertificateException;
 import com.spotify.docker.client.exceptions.DockerException;
 import com.spotify.docker.client.messages.*;
+import org.apache.commons.lang3.StringUtils;
 
 import java.net.URI;
 import java.nio.file.Paths;
@@ -258,11 +259,12 @@ public class DockerClientUtils {
      * @throws InterruptedException
      */
     public static void deleteContainer(String containerId) throws DockerException, InterruptedException {
-
-        //首先要停止容器
-        docker.stopContainer(containerId,0);
-        //然后删除容器
-        docker.removeContainer(containerId);
+        if (StringUtils.isNotEmpty(containerId)) {
+            //首先要停止容器
+            docker.stopContainer(containerId, 0);
+            //然后删除容器
+            docker.removeContainer(containerId);
+        }
     }
 
 

src/main/java/io/renren/modules/dataSet/enumeration/DataSetType.java

@@ -0,0 +1,32 @@
+package io.renren.modules.dataSet.enumeration;
+
+/**
+ * @Author: seamew
+ * @Title: DataSetType
+ * @CreateTime: 2023年09月19日 22:38:00
+ * @Description: 数据集枚举类
+ * @Version: 1.0
+ */
+public enum DataSetType {
+    STATIC_DATASET("dataset", ""),
+    DYNAMIC_DATASET("dydataset", ""),
+    FILE_DATASET("filedataset", "单文件"),
+    DIR_DATASET("dirdataset", "");
+
+    private final String bucketName;
+
+    private final String classificationName;
+
+    DataSetType(String bucketName, String classificationName) {
+        this.bucketName = bucketName;
+        this.classificationName = classificationName;
+    }
+
+    public String getBucketName() {
+        return bucketName;
+    }
+
+    public String getClassificationName() {
+        return classificationName;
+    }
+}

src/main/java/io/renren/modules/dataSet/upload/UpLoadController.java

@@ -2,8 +2,14 @@ package io.renren.modules.dataSet.upload;
 
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.fastjson.JSON;
+import com.google.gson.Gson;
 import io.renren.common.exception.RRException;
+import io.renren.common.utils.MinIoUtils;
 import io.renren.common.utils.R;
+import io.renren.modules.dataSet.enumeration.DataSetType;
+import io.renren.modules.sys.entity.dataset.DataSet;
+import io.renren.modules.sys.service.CategoryService;
+import io.renren.modules.sys.service.DataSetService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.*;
@@ -38,6 +44,13 @@ public class UpLoadController {
      */
     @Autowired
     private UpLoadService upLoadService;
+
+    @Autowired
+    private DataSetService dataSetService;
+
+    @Autowired
+    private CategoryService categoryService;
+
     @PostMapping("upload")
     @ResponseBody
     public R uploadExcel(MultipartFile file) throws IOException {
@@ -50,20 +63,32 @@ public class UpLoadController {
 
     }
 
+    @RequestMapping("uploadFile")
+    public R uploadFile(@RequestPart("file") MultipartFile file, @RequestParam("dataSet") String dataSetJson) throws Exception {
+        MinIoUtils.uploadMultipartFile(file, DataSetType.FILE_DATASET.getBucketName(), file.getOriginalFilename());
+        DataSet dataSet = new Gson().fromJson(dataSetJson, DataSet.class);
+        dataSet.setCreateTime(new Date());
+        dataSet.setCategoryId(categoryService.getByName(DataSetType.FILE_DATASET.getClassificationName()));
+        dataSetService.save(dataSet);
+        return R.ok();
+    }
+
+
     @RequestMapping("uploadcsv")
     public R uploadCSV(MultipartFile file) throws Exception {
         System.out.println(file);
         Map map = upLoadService.uploadCSV(file);
 //        System.out.println(map);
-        return R.ok().put("data",map);
+        return R.ok().put("data", map);
     }
+
     @RequestMapping("getstapre")
-    public R getstaticprecontent(@RequestBody Map map){
+    public R getstaticprecontent(@RequestBody Map map) {
 
         final List getstaticprecontent = upLoadService.getstaticprecontent(map);
 //        System.out.println(datas);
 
 //        subeqpdemoService.getPrecontent(subeqpments);
-        return R.ok().put("data",getstaticprecontent);
+        return R.ok().put("data", getstaticprecontent);
     }
 }

src/main/java/io/renren/modules/sys/controller/VisiWorkflowController.java

@@ -569,6 +569,8 @@ public class VisiWorkflowController extends AbstractController {
                 containertemp.setCommand(command);   //List<String> command
                 containertemp.args(args);  //List<String> args
                 containertemp.setEnv(envlist);   //20211208
+                // 设置代码拉取规则
+                containertemp.setImagePullPolicy("IfNotPresent");
 
                 template.setContainer(containertemp);
                 //添加至workflowTemplate的List<Template> templateslist中

src/main/java/io/renren/modules/sys/controller/algs/algTrainController.java

@@ -18,6 +18,7 @@ import io.renren.modules.sys.service.impl.AlgTrainServiceImpl;
 import io.renren.modules.sys.service.impl.AlgsModelsServiceImpl;
 import io.renren.modules.sys.service.impl.AlgsServiceImpl;
 import io.swagger.models.auth.In;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -521,9 +522,11 @@ public class algTrainController {
         DockerClientUtils.deleteContainer(algTrain.getContainerId());//首先将该容器删除
         FTPUtils.removeDir("algTrain"+algorithmTrainingId);//其次将存储卷删除
         algTrainService.removeById(Long.parseLong(algorithmTrainingId));//数据库中删除
-        String port=algTrain.getIpAddress().substring(algTrain.getIpAddress().length()-5);
-        ServerPort serverPort=new ServerPort(Long.parseLong(port),0);
-        serverPortService.updateServerPort(serverPort);//将端口释放
+        if (StringUtils.isNotEmpty(algTrain.getIpAddress())) {
+            String port = algTrain.getIpAddress().substring(algTrain.getIpAddress().length() - 5);
+            ServerPort serverPort = new ServerPort(Long.parseLong(port), 0);
+            serverPortService.updateServerPort(serverPort);//将端口释放
+        }
         return R.ok();
     }
 

src/main/java/io/renren/modules/sys/controller/dataset/DataSetController.java

@@ -9,9 +9,11 @@ import io.renren.common.utils.R;
 import io.renren.common.validator.ValidatorUtils;
 import io.renren.common.validator.group.AddGroup;
 import io.renren.common.validator.group.UpdateGroup;
+import io.renren.modules.dataSet.enumeration.DataSetType;
 import io.renren.modules.sys.entity.algs.AlgorithmModel;
 import io.renren.modules.sys.entity.algs.FileTest;
 import io.renren.modules.sys.entity.dataset.DataSet;
+import io.renren.modules.sys.service.CategoryService;
 import io.renren.modules.sys.service.DataSetService;
 import io.renren.modules.sys.service.impl.AlgsModelsServiceImpl;
 import lombok.SneakyThrows;
@@ -36,6 +38,9 @@ public class DataSetController {
 
     @Autowired
     DataSetService dataSetService;
+
+    @Autowired
+    private CategoryService categoryService;
     /**
      * 所有列表
      */
@@ -125,24 +130,30 @@ public class DataSetController {
 
         List<DataSet> dataSets = dataSetService.listByIds(Arrays.asList(datasetIds));
         dataSetService.deleteBatch(datasetIds);
-        System.out.println(dataSets);
-        for (DataSet dataset:dataSets
-             ) {
-            MinIoUtils.deleteFile("dataset",dataset.getDatasetName()+".csv");
+        Long singleFile = categoryService.getByName(DataSetType.FILE_DATASET.getClassificationName());
+        Long dir = categoryService.getByName(DataSetType.DIR_DATASET.getClassificationName());
+        for (DataSet dataset : dataSets) {
+            if (dataset.getCategoryId().equals(singleFile)) {
+                MinIoUtils.deleteFile(DataSetType.FILE_DATASET.getBucketName(), dataset.getDatasetName());
+            } else if (dataset.getCategoryId().equals(dir)) {
+                MinIoUtils.deleteFile(DataSetType.DIR_DATASET.getBucketName(), dataset.getDatasetName());
+            } else {
+                MinIoUtils.deleteFile(DataSetType.STATIC_DATASET.getBucketName(), dataset.getDatasetName()+".csv");
+            }
         }
         return R.ok();
     }
 
     @GetMapping("/listAll")
     public R listAll() throws Exception {
-        List<FileTest> list=MinIoUtils.listBucketFiles("dataset");
+        List<FileTest> list=MinIoUtils.listBucketFiles(DataSetType.STATIC_DATASET.getBucketName());
 
         return R.ok().put("list",list);
     }
 
     @GetMapping("/listAllDyDataset")
     public R listAllDyDataset() throws Exception {
-        List<FileTest> list=MinIoUtils.listBucketFiles("dydataset");
+        List<FileTest> list=MinIoUtils.listBucketFiles(DataSetType.DYNAMIC_DATASET.getBucketName());
 
         return R.ok().put("list",list);
     }

src/main/resources/application-dev.yml

@@ -1,12 +1,14 @@
 spring:
     datasource:
+        database-name: hebing_menu
         type: com.alibaba.druid.pool.DruidDataSource
         druid:
             driver-class-name: com.mysql.cj.jdbc.Driver
-            url: jdbc:mysql://localhost:3306/renren_fast?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
+            #43.143.221.128:6033/renren_fast
+            url: jdbc:mysql://82.156.153.135:6033/renren_fast?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
             username: root
-            password: 123456
-#            password: root
+            password: XDUxdu2022
+            #            password: XDUxdu2022
             initial-size: 10
             max-active: 100
             min-idle: 10
@@ -37,16 +39,22 @@ spring:
 
 ##多数据源的配置
 dynamic:
-  datasource:
-    bigData:
-#      url: jdbc:mysql://112.126.68.148:6033/hebing?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
-        url: jdbc:mysql://lab2:6033/bigdata_sgwtest?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
-#        url: jdbc:mysql://localhost:3306/hebing?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
-        username: root
-        password: XDU520bdm
-#        password: root
-#        password: 123456
-        driver-class-name: com.mysql.jdbc.Driver # 3.2.0开始支持SPI可省略此配置
+    datasource:
+        #        bigData:
+        #            #      url: jdbc:mysql://112.126.68.148:6033/hebing?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
+        #            url: jdbc:mysql://43.143.221.128:6033/bigdata_dev?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
+        #            #        url: jdbc:mysql://localhost:3306/hebing?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
+        #            username: root
+        #            password: XDUxdu2022
+        #            #        password: root
+        #            #        password: 123456
+        #            driver-class-name: com.mysql.jdbc.Driver # 3.2.0开始支持SPI可省略此配置
+        heBing:
+            url: jdbc:mysql://82.156.153.135:6033/hebing_menu?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
+            username: root
+            password: XDUxdu2022
+            driver-class-name: com.mysql.cj.jdbc.Driver
+
 server:
     port: 8082
 #      password: 123456
@@ -54,7 +62,7 @@ server:
 
 hbase:
     config:
-        hbase.zookeeper.quorum: 150.158.138.99,49.235.67.21,42.192.195.253
+        hbase.zookeeper.quorum: 43.143.226.39,43.143.224.212,43.143.230.108
         #hbase.zookeeper.quorum: 182.92.79.148,112.126.68.148,123.56.99.12
         hbase.zookeeper.property.clientPort: 2181
         zookeeper.znode.parent: /hbase

src/main/resources/application.yml

@@ -10,7 +10,7 @@ server:
     context-path: /renren-fast
 #  minIO
 minio:
-  endpoint: http://150.158.138.99:9006
+  endpoint: http://43.143.237.223:9000
   bucket: test0706
   access-key: minio
   secret-key: minio123
@@ -18,38 +18,39 @@ minio:
 #docker远程连接
 docker:
   #url: https://192.168.25.101:2376
-  url: https://150.158.138.99:2376
-remote-docker: 150.158.138.99
+  url: https://43.143.237.223:2376
+remote-docker: 43.143.237.223
 #上传文件到远程服务器地址,用户名及密码,上传文件的基础路径
-host: 150.158.138.99
+host: 43.143.237.223
 host-username: root
-host-password: XDU520bdm
+host-password: XDUxdu2022
+#host-basepath: D:/aiplat-mkcloud/aiplat/uploadFile
 host-basepath: /opt/uploadFile
 host-port: 22
 #连接远程服务器是否需要秘钥
-key-needed: true
+key-needed: false
 #如果需要秘钥，则将秘钥地址填写在此
-key-location: D:/Program Files/secretKey/lab1
+key-location: D:/aiplat-mkcloud/aiplat/docker_ca
 
 #docker证书存放路径
-docker_ca: D:/aiplat/school_ca
+docker_ca: D:/aiplat-mkcloud/aiplat/docker_ca
 
 #minio暂存文件地址
-tempFileLocation: D:/aiplat/test0708/testPython.py
+tempFileLocation: D:/aiplat-mkcloud/test0708/testPython.py
 
 #获取kafka实时数据ip与端口
-realTimeKafka1: 10.168.57.10:9092
-realTimeKafka2: 10.168.57.11:9092
-realTimeKafka3: 10.168.57.12:9092
+realTimeKafka1: 43.143.230.108:9092
+realTimeKafka2: 43.143.224.212:9092
+realTimeKafka3: 43.143.226.39:9092
 
 #算法结果入库模板ip
-resultToDB: 49.235.67.21
+resultToDB: 43.143.226.39
 
 #数据预处理数据集存放文件夹
-datasetLocation: D:/aiplat/demoCSV
+datasetLocation: D:/aiplat-mkcloud/aiplat/demoCSV
 
 #数据预处理python文件存放文件夹
-dataPreProcessLocation: D:/aiplat/dataPreProcess
+dataPreProcessLocation: D:/aiplat-mkcloud/aiplat/dataPreProcess
 
 spring:
   # 环境 dev|test|prod
@@ -80,8 +81,10 @@ spring:
   mvc:
     throw-exception-if-no-handler-found: true
   resources:
-   add-mappings: true
+    add-mappings: true
 
+#层级相关数据库
+cengji: hebing_menu
 
 
 #mybatis
@@ -120,8 +123,8 @@ renren:
 
 #Argo
 argo:
-  basepath: https://42.192.195.253:30865
-  minioendpoint: http://42.192.195.253:30253
+  basepath: https://43.143.237.223:31691
+  minioendpoint: http://43.143.237.223:31352
   minioAccess-key: admin
   #  minioAccess-key: vzZBapBXei60zI1t6oPl
   minioSecret-key: password
@@ -133,8 +136,11 @@ argo:
   sparkYarnJARS: hdfs://150.158.138.99:8020
 #kubenetes  Argo部分获取pod日志
 Akubenetes:
-  basepath: https://42.192.195.253:6443
-  kubeconfig-path: D:/zhang/IdeaProjects/20220322/config
+  basepath: https://43.143.237.223:30880
+  kubeconfig-path: D:/aiplat-mkcloud/aiplat/akubenetes/config
+
+#griffin数据探查服务
+griffin: http://43.143.224.212:8091