数据探查修改

src/main/java/io/renren/common/utils/ToEnglish.java

@@ -41,7 +41,7 @@ public class ToEnglish {
             e.printStackTrace();
 //    		Log.v(TAG, "BadHanyuPinyinOutputFormatCombination");
         }
-        return output;
+        return StringUtils.strSpecialFilter(output) ;
     }
 
 }

src/main/java/io/renren/modules/snoop/controller/SnoopController.java

@@ -0,0 +1,67 @@
+package io.renren.modules.snoop.controller;
+
+import io.renren.common.utils.PageUtils;
+import io.renren.modules.snoop.entity.SnoopAndSnoopValueEntity;
+import io.renren.modules.snoop.entity.SnoopEntity;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+import io.renren.modules.snoop.service.SnoopService;
+import io.renren.modules.sys.controller.AbstractController;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import io.renren.common.utils.R;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/snoop")
+public class SnoopController extends AbstractController {
+
+    @Autowired
+    private SnoopService snoopService;
+
+    /**
+     * 根据名称进行搜索
+     */
+    @RequestMapping("/selectByName")
+    public R selectByName(@RequestParam(defaultValue = "1") Integer page,
+                          @RequestParam(defaultValue = "100") Integer size,
+                          @RequestParam(value = "name") String name){
+
+        List<SnoopEntity> snoopEntityList = snoopService.selectByName(page, size, name);
+        Integer total = snoopService.getCountByName(name);
+        return R.ok().put("count", total).put("snoopList", snoopEntityList);
+    }
+
+    /**
+     * 探查列表
+     */
+    @RequestMapping("/list")
+    public R list(@RequestParam Map<String, Object> params){
+        PageUtils page = snoopService.queryPage(params);
+        return R.ok().put("page", page);
+    }
+
+    /**
+     * 删除探查
+     */
+    @RequestMapping("/delete/{measureID}")
+    public R delete(@PathVariable("measureID") Integer snoopIds){
+        snoopService.deleteBatch(snoopIds);
+        return R.ok();
+    }
+
+    /**
+     * 增加探查
+     * snoopEntity：探查名、备注、Hbase表名
+     * snoopValueEntityList[i]：探查哪一列、探查该列的哪种类型
+     */
+    @RequestMapping("/save")
+    public R save(@RequestBody SnoopAndSnoopValueEntity snoopAndSnoopValueEntity){
+
+        snoopAndSnoopValueEntity.getHbaseName();
+        snoopAndSnoopValueEntity.getRemark();
+        snoopService.saveMeasure(snoopAndSnoopValueEntity);
+        return R.ok();
+    }
+}

src/main/java/io/renren/modules/snoop/controller/SnoopValueController.java

@@ -0,0 +1,32 @@
+package io.renren.modules.snoop.controller;
+
+import io.renren.common.utils.PageUtils;
+import io.renren.common.utils.R;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+import io.renren.modules.snoop.service.SnoopValueService;
+import io.renren.modules.sys.controller.AbstractController;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/snoopvalue")
+public class SnoopValueController extends AbstractController {
+
+    @Autowired
+    private SnoopValueService snoopValueService;
+
+    /**
+     *
+     */
+    @RequestMapping("/selectBySnoopId")
+    public R selectBySnoopId(@RequestParam Integer snoopId){
+
+        List<SnoopValueEntity> snoopValueList = snoopValueService.selectBySnoopId(snoopId);
+        return R.ok().put("snoopvaluelist", snoopValueList);
+    }
+}

src/main/java/io/renren/modules/snoop/dao/SnoopDao.java

@@ -0,0 +1,16 @@
+package io.renren.modules.snoop.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import io.renren.modules.snoop.entity.SnoopEntity;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+@Mapper
+public interface SnoopDao extends BaseMapper<SnoopEntity> {
+
+    List<SnoopEntity> selectByName(@Param("page")Integer page, @Param("size")Integer size, @Param("name")String name);
+
+    Integer getCountByName(@Param("name")String name);
+}

src/main/java/io/renren/modules/snoop/dao/SnoopValueDao.java

@@ -0,0 +1,10 @@
+package io.renren.modules.snoop.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+import org.apache.ibatis.annotations.Mapper;
+
+@Mapper
+public interface SnoopValueDao extends BaseMapper<SnoopValueEntity> {
+
+}

src/main/java/io/renren/modules/snoop/entity/ColumnRuleEntity.java

@@ -0,0 +1,19 @@
+package io.renren.modules.snoop.entity;
+
+import lombok.Data;
+import org.supercsv.cellprocessor.constraint.StrNotNullOrEmpty;
+
+import java.io.Serializable;
+
+/**
+ * 接收前端传递过来的列和规则
+ */
+@Data
+public class ColumnRuleEntity implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private String measureColumn;
+
+    private String measureType;
+}

src/main/java/io/renren/modules/snoop/entity/SnoopAndSnoopValueEntity.java

@@ -0,0 +1,23 @@
+package io.renren.modules.snoop.entity;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * 接收前端传递过来的snoop和snoop_value表参数
+ */
+@Data
+public class SnoopAndSnoopValueEntity implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private String snoopName;
+
+    private String hbaseName;
+
+    private String remark;
+
+    private List<ColumnRuleEntity> snoopValueEntityList;
+}

src/main/java/io/renren/modules/snoop/entity/SnoopEntity.java

@@ -0,0 +1,39 @@
+package io.renren.modules.snoop.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.io.Serializable;
+
+/**
+ * 数据探查表
+ */
+@Data
+@TableName("snoop")
+public class SnoopEntity implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * 探查id
+     */
+    @TableId(type = IdType.AUTO)
+    private Integer id;
+    /**
+     * 探查名称
+     */
+    private String snoopName;
+    /**
+     * 探查类型
+     */
+    private String snoopType;
+    /**
+     * 备注
+     */
+    private String remark;
+    /**
+     * 相关联hbase表名称
+     */
+    private String hbaseName;
+}

src/main/java/io/renren/modules/snoop/entity/SnoopValueEntity.java

@@ -0,0 +1,39 @@
+package io.renren.modules.snoop.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.io.Serializable;
+
+/**
+ * 探查值表
+ */
+@Data
+@TableName("snoop_value")
+public class SnoopValueEntity implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * 探查值id
+     */
+    @TableId(type = IdType.AUTO)
+    private Integer id;
+    /**
+     * 相对应的探查id
+     */
+    private Integer snoopId;
+    /**
+     * 探查类型：最大、最小、平均...
+     */
+    private String measureType;
+    /**
+     * 探查值
+     */
+    private String measureValue;
+    /**
+     * 要探测的列的名称
+     */
+    private String measureColumn;
+}

src/main/java/io/renren/modules/snoop/service/SnoopService.java

@@ -0,0 +1,23 @@
+package io.renren.modules.snoop.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import io.renren.common.utils.PageUtils;
+import io.renren.modules.snoop.entity.SnoopAndSnoopValueEntity;
+import io.renren.modules.snoop.entity.SnoopEntity;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+
+import java.util.List;
+import java.util.Map;
+
+public interface SnoopService extends IService<SnoopEntity> {
+    List<SnoopEntity> selectByName(Integer page, Integer size, String name);
+
+    Integer getCountByName(String name);
+
+    PageUtils queryPage(Map<String, Object> params);
+
+    void deleteBatch(Integer snoopIds);
+
+    void saveMeasure(SnoopAndSnoopValueEntity snoopAndSnoopValueEntity);
+
+}

src/main/java/io/renren/modules/snoop/service/SnoopValueService.java

@@ -0,0 +1,10 @@
+package io.renren.modules.snoop.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+
+import java.util.List;
+
+public interface SnoopValueService extends IService<SnoopValueEntity> {
+    List<SnoopValueEntity> selectBySnoopId(Integer snoopId);
+}

src/main/java/io/renren/modules/snoop/service/impl/SnoopServiceImpl.java

@@ -0,0 +1,267 @@
+package io.renren.modules.snoop.service.impl;
+
+import afu.org.checkerframework.checker.igj.qual.I;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import io.renren.common.utils.PageUtils;
+import io.renren.common.utils.Query;
+import io.renren.common.utils.ToEnglish;
+import io.renren.modules.dataSet.service.DynamicSystemService;
+import io.renren.modules.levelManage.entity.LevelEntity;
+import io.renren.modules.snoop.dao.SnoopDao;
+import io.renren.modules.snoop.entity.ColumnRuleEntity;
+import io.renren.modules.snoop.entity.SnoopAndSnoopValueEntity;
+import io.renren.modules.snoop.entity.SnoopEntity;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+import io.renren.modules.snoop.service.SnoopService;
+import io.renren.modules.snoop.service.SnoopValueService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.*;
+
+@Service("snoopService")
+public class SnoopServiceImpl extends ServiceImpl<SnoopDao, SnoopEntity> implements SnoopService {
+
+    @Resource
+    private  SnoopDao snoopDao;
+    @Autowired
+    private DynamicSystemService dynamicSystemService;
+    @Autowired
+    private SnoopValueService snoopValueService;
+
+    @Override
+    public List<SnoopEntity> selectByName(Integer page, Integer size, String name) {
+        if(page != null && size != null){
+            page = (page - 1) * size;
+        }
+
+        return snoopDao.selectByName(page, size, name);
+    }
+
+    @Override
+    public Integer getCountByName(String name) {
+        return snoopDao.getCountByName(name);
+    }
+
+    @Override
+    public PageUtils queryPage(Map<String, Object> params) {
+        IPage<SnoopEntity> page = this.page(
+                new Query<SnoopEntity>().getPage(params),
+                new QueryWrapper<SnoopEntity>()
+        );
+
+        return new PageUtils(page);
+    }
+
+    @Override
+    public void deleteBatch(Integer snoopIds) {
+        // 删除snoop表中的部分
+        this.removeByIds(Arrays.asList(snoopIds));
+
+        // 删除snoop_value表中与snoop对应的部分
+        QueryWrapper<SnoopValueEntity> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("snoop_id", snoopIds);
+        snoopValueService.remove(queryWrapper);
+
+    }
+
+    @Override
+    public void saveMeasure(SnoopAndSnoopValueEntity snoopAndSnoopValueEntity) {
+
+        // 保存snoopEntity
+        SnoopEntity snoopEntity = new SnoopEntity();
+        snoopEntity.setHbaseName(snoopAndSnoopValueEntity.getHbaseName());
+        snoopEntity.setRemark(snoopAndSnoopValueEntity.getRemark());
+        snoopEntity.setSnoopName(snoopAndSnoopValueEntity.getSnoopName());
+        this.save(snoopEntity);
+
+        // 得到Hbase表中数据 List<Map<String, String>>
+        List<Map<String, String>> equmentAll = dynamicSystemService.getEqumentLimit(snoopEntity.getHbaseName(),100);
+
+        // 根据每一个snoopValueEntity 计算出 每列的值
+        for(ColumnRuleEntity columnRuleEntity : snoopAndSnoopValueEntity.getSnoopValueEntityList()){
+            String measureType = columnRuleEntity.getMeasureType();
+            String measureColumn = columnRuleEntity.getMeasureColumn();
+
+            List<String> allVaules = new ArrayList<>();
+            // 列名 ——> 拼音+after
+            String pinYinmeasureColumn = ToEnglish.getPinYin(measureColumn);
+            pinYinmeasureColumn += "_after";
+            for(Map<String, String> item : equmentAll){
+                allVaules.add(item.get(pinYinmeasureColumn));
+            }
+
+            // 进行计算
+            String res = computerValues(allVaules, measureType);
+
+            // 保存每一个snoopValueEntity
+            SnoopValueEntity snoopValueEntity = new SnoopValueEntity();
+            snoopValueEntity.setMeasureType(measureType);
+            snoopValueEntity.setMeasureColumn(measureColumn);
+            snoopValueEntity.setMeasureValue(res);
+            snoopValueEntity.setSnoopId(snoopEntity.getId());
+            snoopValueService.save(snoopValueEntity);
+        }
+
+    }
+
+    private String computerValues(List<String> allVaules, String measureType) {
+
+        String res = "";
+
+        if(measureType.equals("Null Count")){
+            res = getNull(allVaules);
+        }else if(measureType.equals("Total Count")){
+            res = getTotal(allVaules);
+        }else if(measureType.equals("Maximum")){
+            res = getMax(allVaules);
+        }else if(measureType.equals("Minimum")){
+            res = getMin(allVaules);
+        }else if(measureType.equals("Average")){
+            res = getAverage(allVaules);
+        }else if(measureType.equals("Mode")){
+            res = getMode(allVaules);
+        }
+        return res;
+    }
+
+    /**
+     * 得到最大值
+     * @param allVaules
+     * @return
+     */
+    private String getMax(List<String> allVaules){
+        Double max = Double.MIN_VALUE;
+        List<Double> list = new ArrayList<>();
+        for(String str : allVaules){
+            try{
+                if(str == "NULL"){
+                    continue;
+                }else{
+                    double num = Double.parseDouble(str);
+                    list.add(num);
+                }
+
+            }catch (Exception e){
+                return "数据存在异常值，无法统计";
+            }
+        }
+        for(Double item : list){
+            if(item > max){
+                max = item;
+            }
+        }
+        return String.valueOf(max);
+    }
+
+    /**
+     * 得到最小值
+     * @param allVaules
+     * @return
+     */
+    private String getMin(List<String> allVaules){
+        Double min = Double.MAX_VALUE;
+        List<Double> list = new ArrayList<>();
+        for(String str : allVaules){
+            try{
+                if(str == "NULL"){
+                    continue;
+                }else{
+                    double num = Double.parseDouble(str);
+                    list.add(num);
+                }
+
+            }catch (Exception e){
+                return "数据存在异常值，无法统计";
+            }
+        }
+        for(Double item : list){
+            if(item < min){
+                min = item;
+            }
+        }
+        return String.valueOf(min);
+    }
+
+    /**
+     * 得到平均值
+     * @param allVaules
+     * @return
+     */
+    private String getAverage(List<String> allVaules){
+        double sum = 0;
+        List<Double> list = new ArrayList<>();
+        for(String str : allVaules){
+            try{
+                if(str == "NULL"){
+                    continue;
+                }else {
+                    double num = Double.parseDouble(str);
+                    list.add(num);
+                }
+            }catch (Exception e){
+                return "数据存在异常值，无法统计";
+            }
+        }
+        for(Double item : list){
+            sum += item;
+        }
+        sum = sum / (double) list.size();
+        return String.valueOf(sum);
+
+    }
+
+    /**
+     * 得到数据总数
+     * @param allVaules
+     * @return
+     */
+    private String getTotal(List<String> allVaules) {
+        int size = allVaules.size();
+        return String.valueOf(size);
+    }
+
+    /**
+     * 得到空值数量
+     * @param allVaules
+     * @return
+     */
+    private String getNull(List<String> allVaules) {
+        int countNull = 0;
+        for(String str : allVaules){
+            if(str.equals("NULL")){
+                countNull++;
+            }
+        }
+        return String.valueOf(countNull);
+    }
+
+    /**
+     * 众数
+     * @param allVaules
+     * @return
+     */
+    private String getMode(List<String> allVaules){
+        Map<String, Integer> countMap = new HashMap<>();
+
+        // 计算每个字符串的出现次数
+        for (String value : allVaules) {
+            countMap.put(value, countMap.getOrDefault(value, 0) + 1);
+        }
+
+        // 找到出现次数最多的字符串
+        String mode = null;
+        int maxCount = 0;
+        for (Map.Entry<String, Integer> entry : countMap.entrySet()) {
+            if (entry.getValue() > maxCount) {
+                mode = entry.getKey();
+                maxCount = entry.getValue();
+            }
+        }
+
+        return mode;
+    }
+}

src/main/java/io/renren/modules/snoop/service/impl/SnoopValueServiceImpl.java

@@ -0,0 +1,22 @@
+package io.renren.modules.snoop.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import io.renren.modules.snoop.dao.SnoopValueDao;
+import io.renren.modules.snoop.entity.SnoopValueEntity;
+import io.renren.modules.snoop.service.SnoopValueService;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service("snoopValueService")
+public class SnoopValueServiceImpl extends ServiceImpl<SnoopValueDao, SnoopValueEntity> implements SnoopValueService {
+
+    @Override
+    public List<SnoopValueEntity> selectBySnoopId(Integer snoopId) {
+        QueryWrapper<SnoopValueEntity> wp = new QueryWrapper<>();
+        wp.eq("snoop_id", snoopId);
+        List<SnoopValueEntity> list = this.list(wp);
+        return list;
+    }
+}

src/main/resources/mapper/snoop/SnoopMapper.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="io.renren.modules.snoop.dao.SnoopDao">
+
+    <select id="selectByName" resultType="io.renren.modules.snoop.entity.SnoopEntity">
+        SELECT * FROM snoop WHERE snoop_name like concat('%',#{name},'%') limit #{page}, #{size};
+    </select>
+
+    <select id="getCountByName" resultType="java.lang.Integer">
+        SELECT count(*) from snoop
+        WHERE name like concat('%',#{name},'%');
+    </select>
+</mapper>

src/main/resources/mapper/snoop/SnoopValueMapper.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="io.renren.modules.snoop.dao.SnoopValueDao">
+
+
+</mapper>