Spark-2.3.2【SparkStreaming+SparkSQL-实时仪表盘应用】
应用场景：实时仪表盘（即大屏），每个集团下有多个mall，每个mall下包含多家shop，需实时计算集团下各mall及其shop的实时销售分析（区域、业态、店铺TOP、总销售额等指标）并提供可视化展现，之前时候一直在Strom实现，现在改为Spark2.3.2实现。
1、数据源：首先数据源来自于MQ、Socket、Flume和DFS等，一般Kafka、RocketMQ等居多，此处示例代码用的是RocketMQ；
2、实时计算框架：Storm（实时计算，Spout发射Tuple到各个Bolt，来一条即处理一条，一百毫秒以内的延迟）、SparkStreaming（准实时计算，基于微批次的实时计算，即一定时间段内的micro batch，每个micro batch的结构为DStream，底层是RDD）；
3、此处Spark Streaming准实时处理应用流程：1、RocketMQ --> 2、SparkStreaming --> 3、SparkSQL（Parquet） --> 4、Redis（大屏使用） && HDFS（Hive数仓ODS层，ODS->DW[DWD-DWS]-DM）；
4、系统设计
5、代码如下（4.3是Spark Streaming实现，复制粘贴即可运行）：
- 5.1、RocketMQConfig类（用于配置和构建MQ消费者）
- 5.2、SparkStreaming自定义RocketMQ接收器（可靠的接收器）
- 5.3、SparkStreaming销售分析实时计算
- 5.4、账单实体类
- 5.5、BCD即账单裁剪后的实体类（减少数据量传输即降低节点间的序列化和反序列化开销）
- 5.6、SparkSQL销售分析业务实现类
5.1、RocketMQConfig类（用于配置和构建MQ消费者）
package com.mengyao.graph.etl.apps.commons.datasource.mq.receiver; import org.apache.commons.lang.Validate; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.remoting.common.RemotingUtil; import java.util.HashMap; import java.util.UUID; /** * RocketMQConfig for Consumer * @author mengyao * */ public class RocketMQConfig { // ------- the following is for common usage ------- /** * RocketMq name server address */ public static final String NAME_SERVER_ADDR = "nameserver.addr"; // Required public static final String CLIENT_NAME = "client.name"; public static final String CLIENT_IP = "client.ip"; public static final String DEFAULT_CLIENT_IP = RemotingUtil.getLocalAddress(); public static final String CLIENT_CALLBACK_EXECUTOR_THREADS = "client.callback.executor.threads"; public static final int DEFAULT_CLIENT_CALLBACK_EXECUTOR_THREADS = Runtime.getRuntime().availableProcessors();; public static final String NAME_SERVER_POLL_INTERVAL = "nameserver.poll.interval"; public static final int DEFAULT_NAME_SERVER_POLL_INTERVAL = 30000; // 30 seconds public static final String BROKER_HEART_BEAT_INTERVAL = "brokerserver.heartbeat.interval"; public static final int DEFAULT_BROKER_HEART_BEAT_INTERVAL = 30000; // 30 seconds // ------- the following is for push consumer mode ------- /** * RocketMq consumer group */ public static final String CONSUMER_GROUP = "consumer.group"; // Required /** * RocketMq consumer topic */ public static final String CONSUMER_TOPIC = "consumer.topic"; // Required public static final String CONSUMER_TAG = "consumer.tag"; public static final String DEFAULT_TAG = "*"; public static final String CONSUMER_OFFSET_RESET_TO = "consumer.offset.reset.to"; public static final String CONSUMER_OFFSET_LATEST = "latest"; public static final String CONSUMER_OFFSET_EARLIEST = "earliest"; public static final String CONSUMER_OFFSET_TIMESTAMP = "timestamp"; public static final String CONSUMER_MESSAGES_ORDERLY = "consumer.messages.orderly"; public static final String CONSUMER_OFFSET_PERSIST_INTERVAL = "consumer.offset.persist.interval"; public static final int DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL = 5000; // 5 seconds public static final String CONSUMER_MIN_THREADS = "consumer.min.threads"; public static final int DEFAULT_CONSUMER_MIN_THREADS = 20; public static final String CONSUMER_MAX_THREADS = "consumer.max.threads"; public static final int DEFAULT_CONSUMER_MAX_THREADS = 64; // ------- the following is for reliable Receiver ------- public static final String QUEUE_SIZE = "spout.queue.size"; public static final int DEFAULT_QUEUE_SIZE = 500; public static final String MESSAGES_MAX_RETRY = "spout.messages.max.retry"; public static final int DEFAULT_MESSAGES_MAX_RETRY = 3; public static final String MESSAGES_TTL = "spout.messages.ttl"; public static final int DEFAULT_MESSAGES_TTL = 300000; // 5min // ------- the following is for pull consumer mode ------- /** * Maximum rate (number of records per second) at which data will be read from each RocketMq partition , * and the default value is "-1", it means consumer can pull message from rocketmq as fast as the consumer can. * Other that, you also enables or disables Spark Streaming's internal backpressure mechanism by the config * "spark.streaming.backpressure.enabled". */ public static final String MAX_PULL_SPEED_PER_PARTITION = "pull.max.speed.per.partition"; /** * To pick up the consume speed, the consumer can pull a batch of messages at a time. And the default * value is "32" */ public static final String PULL_MAX_BATCH_SIZE = "pull.max.batch.size"; /** * pull timeout for the consumer, and the default time is "3000". */ public static final String PULL_TIMEOUT_MS = "pull.timeout.ms"; // the following configs for consumer cache public static final String PULL_CONSUMER_CACHE_INIT_CAPACITY = "pull.consumer.cache.initialCapacity"; public static final String PULL_CONSUMER_CACHE_MAX_CAPACITY = "pull.consumer.cache.maxCapacity"; public static final String PULL_CONSUMER_CACHE_LOAD_FACTOR = "pull.consumer.cache.loadFactor"; public static void buildConsumerConfigs(HashMap<String, String> props, DefaultMQPushConsumer consumer) { buildCommonConfigs(props, consumer); String group = props.get(CONSUMER_GROUP); Validate.notEmpty(group); consumer.setConsumerGroup(group); consumer.setPersistConsumerOffsetInterval(getInteger(props, CONSUMER_OFFSET_PERSIST_INTERVAL, DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL)); consumer.setConsumeThreadMin(getInteger(props, CONSUMER_MIN_THREADS, DEFAULT_CONSUMER_MIN_THREADS)); consumer.setConsumeThreadMax(getInteger(props, CONSUMER_MAX_THREADS, DEFAULT_CONSUMER_MAX_THREADS)); String initOffset = props.get(CONSUMER_OFFSET_RESET_TO) != null ? props.get(CONSUMER_OFFSET_RESET_TO) : CONSUMER_OFFSET_LATEST; switch (initOffset) { case CONSUMER_OFFSET_EARLIEST: consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); break; case CONSUMER_OFFSET_LATEST: consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); break; case CONSUMER_OFFSET_TIMESTAMP: consumer.setConsumeTimestamp(initOffset); break; default: consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); } String topic = props.get(CONSUMER_TOPIC); Validate.notEmpty(topic); try { consumer.subscribe(topic, props.get(CONSUMER_TAG) != null ? props.get(CONSUMER_TAG) : DEFAULT_TAG); } catch (MQClientException e) { throw new IllegalArgumentException(e); } } public static void buildCommonConfigs(HashMap<String, String> props, ClientConfig client) { String namesvr = props.get(NAME_SERVER_ADDR); Validate.notEmpty(namesvr); client.setNamesrvAddr(namesvr); client.setClientIP(props.get(CLIENT_IP) != null ? props.get(CLIENT_IP) : DEFAULT_CLIENT_IP); // use UUID for client name by default String defaultClientName = UUID.randomUUID().toString(); client.setInstanceName(props.get(CLIENT_NAME) != null ? props.get(CLIENT_NAME) : defaultClientName); client.setClientCallbackExecutorThreads(getInteger(props, CLIENT_CALLBACK_EXECUTOR_THREADS, DEFAULT_CLIENT_CALLBACK_EXECUTOR_THREADS)); client.setPollNameServerInterval(getInteger(props, NAME_SERVER_POLL_INTERVAL, DEFAULT_NAME_SERVER_POLL_INTERVAL)); client.setHeartbeatBrokerInterval(getInteger(props, BROKER_HEART_BEAT_INTERVAL, DEFAULT_BROKER_HEART_BEAT_INTERVAL)); } public static int getInteger(HashMap<String, String> props, String key, int defaultValue) { return Integer.parseInt(props.get(key) != null ? props.get(key) : String.valueOf(defaultValue)); } public static boolean getBoolean(HashMap<String, String> props, String key, boolean defaultValue) { return Boolean.parseBoolean(props.get(key) != null ? props.get(key) : String.valueOf(defaultValue)); } }
4.2、SparkStreaming自定义RocketMQ接收器（可靠的接收器）
package com.mengyao.graph.etl.apps.commons.datasource.mq.receiver; import org.apache.activemq.util.ByteArrayInputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; //import org.apache.commons.lang3.SerializationUtils; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.MQPushConsumer; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import com.gooagoo.entity.mq.bill.MQBillMessage; import com.mengyao.graph.etl.apps.commons.beans.ods.fmt.BillInfoFmt; import com.mengyao.graph.etl.apps.commons.beans.ods.fmt.ConvertTool; import com.mengyao.graph.etl.apps.dashboard.beans.BCD; import java.io.IOException; import java.io.ObjectInputStream; import java.util.HashMap; import java.util.List; /** * RocketMQ Receiver * @author mengyao * */ public class RocketMQReceiver extends Receiver<BCD> { /** * */ private static final long serialVersionUID = 2274826339951693341L; private MQPushConsumer consumer; private boolean ordered; private HashMap<String, String> conf; public RocketMQReceiver(HashMap<String, String> conf, StorageLevel storageLevel) { super(storageLevel); this.conf = conf; } @Override public void onStart() { Validate.notEmpty(conf, "Consumer properties can not be empty"); ordered = RocketMQConfig.getBoolean(conf, RocketMQConfig.CONSUMER_MESSAGES_ORDERLY, true); consumer = new DefaultMQPushConsumer(); RocketMQConfig.buildConsumerConfigs(conf, (DefaultMQPushConsumer)consumer); if (ordered) { consumer.registerMessageListener(new MessageListenerOrderly() { @Override public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) { if (process(msgs)) { return ConsumeOrderlyStatus.SUCCESS; } else { return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; } } }); } else { consumer.registerMessageListener(new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) { if (process(msgs)) { return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } else { return ConsumeConcurrentlyStatus.RECONSUME_LATER; } } }); } try { consumer.start(); System.out.println("==== RocketMQReceiver start ===="); } catch (MQClientException e) { e.printStackTrace(); throw new RuntimeException(e); } } public boolean process(List<MessageExt> msgs) { if (msgs.isEmpty()) { System.out.println("==== Msgs is null! ===="); return true; } System.out.println("==== receiver msgs: "+msgs.size()+" record. ===="); try { for (MessageExt messageExt : msgs) { //MQBillMessage message = SerializationUtils.deserialize(messageExt.getBody()); MQBillMessage message = deserialize(messageExt.getBody()); if (null != message) { BillInfoFmt billFmt = ConvertTool.convertGagBillToOdsBillFmt(message.getData()); if (validBillType(billFmt)) {
/**
* this.store(BCD) 简单的接收一条存储一条，缺点是没有确认机制，不具备容错保证，会出现数据丢失。优点则是效率更高。
* this.store(Iterator<BCD>)阻塞调用，当接收到的记录都存储到Spark后才会确认成功，当接收方采用复制（默认存储级别为复制）则在复制完成后确认成功，但在缓冲中的数据不被保证，会被重新发送。具备容错保证，可确保数据0丢失。
*/ this.store(new BCD(billFmt.getId(), billFmt.getReceivableAmount(), billFmt.getSaleTime(), billFmt.getBillType(), billFmt.getShopId(), billFmt.getShopEntityId())); } billFmt=null; message.setData(null); message = null; } else { System.out.println("==== receiver msg is:"+messageExt.getBody()+", deserialize faild. ===="); } } return true; } catch (Exception e) { e.printStackTrace(); return false; } } /** * 验证账单类型是否为1、3、6 * @param bean * @return */ static boolean validBillType(BillInfoFmt bean) { if (null == bean) { System.out.println("==== BillBean is null! ===="); return false; } String billType = bean.getBillType(); if (StringUtils.isEmpty(billType)) { System.out.println("==== BillBean.billType is null! ===="); return false; } String billTypeTrim = billType.trim(); return billTypeTrim.equals("1") || billType.equals("3") || billType.equals("6"); } @Override public void onStop() { consumer.shutdown(); System.out.println("==== RocketMQReceiver stop ===="); } private static MQBillMessage deserialize(byte[] body) throws Exception { if (body == null) { throw new IllegalArgumentException("The byte array must not be null"); } MQBillMessage message = null; ObjectInputStream in = null; ByteArrayInputStream bais = null; try { bais = new ByteArrayInputStream(body); in = new ObjectInputStream(bais); message = (MQBillMessage) in.readObject(); } catch (final ClassNotFoundException ex) { ex.printStackTrace(); throw ex; } catch (final IOException ex) { ex.printStackTrace(); throw ex; } finally { try { if (bais != null) { bais.close(); } if (in != null) { in.close(); } } catch (final IOException ex) { // ignore close exception } } return message; } }
4.3、SparkStreaming销售分析实时计
package com.mengyao.graph.etl.apps.commons.datasource.bill; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.StorageLevels; import org.apache.spark.api.java.function.Function0; import org.apache.spark.broadcast.Broadcast; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.apache.spark.streaming.Durations; import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; import org.apache.spark.streaming.api.java.JavaStreamingContext; import org.apache.spark.util.LongAccumulator; import org.apache.spark.util.SizeEstimator; import org.apache.spark.streaming.Time; import com.mengyao.graph.etl.apps.commons.beans.dim.RuianMall; import com.mengyao.graph.etl.apps.commons.datasource.mq.receiver.RocketMQConfig; import com.mengyao.graph.etl.apps.commons.datasource.mq.receiver.RocketMQReceiver; import com.mengyao.graph.etl.apps.dashboard.beans.BCD; import com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService; /** * BillConsumer 大屏实时计算 * 1、hdfs dfs -rm -r hdfs://bd001:8020/data/consumer/bill/checkpoint/* * 2、hdfs dfs -rm -r hdfs://bd001:8020/data/dashboard/ruian/sdt=curTime * 3、spark-submit --class com.mengyao.graph.etl.apps.commons.datasource.bill.BillConsumer --master yarn --deploy-mode cluster --driver-memory 4g --executor-cores 6 --executor-memory 5g --queue default --verbose data-graph-etl.jar * * @author mengyao * */ public class BillConsumer { private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMD = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMdd")); private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMDHMS = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMddHHmmss")); private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMDHMSS = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMddHHmmssSSS")); private static final String BASE_PATH = "hdfs://bd001:8020/data/dashboard/ruian/"; private static final String TMP_PATH = "/merge"; private static String appName = "BillConsumer"; private static String logLevel = "ERROR"; public static void main(String[] args) { args = new String[] {"hdfs://bd001:8020/data/consumer/bill/checkpoint", "10", "base.mq.goo.com:9877", "Bill", "bill_dev_group_00013"}; if (args.length < 5) { System.err.println("Usage: "+appName+" <checkpointDir> <milliseconds> <namesrvAddr> <groupId> <topic>"); System.exit(1); } String checkPointDir = args[0]; int second = Integer.parseInt(args[1]); String namesrv = args[2]; Validate.notNull(namesrv, "RocketMQ namesrv not null!"); String topic = args[3]; Validate.notNull(topic, "RocketMQ topic not null!"); String group = args[4]; Validate.notNull(group, "RocketMQ group not null!"); Function0<JavaStreamingContext> jscFunc = () -> createJavaSparkStreamingContext(checkPointDir, second, namesrv, topic, group); JavaStreamingContext jssc = JavaStreamingContext.getOrCreate(checkPointDir, jscFunc, new Configuration()); jssc.sparkContext().setLogLevel(logLevel); try { jssc.start(); jssc.awaitTermination(); } catch (InterruptedException e) { e.printStackTrace(); } finally { jssc.stop(false, true);//关闭StreamingContext但不关闭SparkContext，同时等待数据处理完成 } } /** * 获取当前时间yyyyMMdd，匹配账单数据saleTime * @param curTime * @return */ static String getCurrentDate(long curTime) { return FORMATTER_YMD.get().format(new Date(curTime)); } /** * 打印bill RDD<BCD>的分区及占用空间大小，debug方法 * @param rdd * @param time */ static void printStream(JavaRDD<BCD> rdd, Time time) {rdd.id(); System.out.println("==== time: "+FORMATTER_YMDHMSS.get().format(new Date(time.milliseconds()))+", rdd: partitions="+rdd.getNumPartitions()+", space="+SizeEstimator.estimate(rdd)/1048576+"mb"); } /** * 合并parquet小文件 * @param session * @param dfsDir */ static void mergeSmallFiles(Dataset<Row> fulls, SparkSession session, String dfsDir) { fulls.coalesce(1).write().mode(SaveMode.Overwrite).parquet(BASE_PATH+TMP_PATH); session.read().parquet(BASE_PATH+TMP_PATH).coalesce(1).write().mode(SaveMode.Overwrite).parquet(dfsDir); } /** * 创建DFS Dir * @param session * @param dfsDir */ static void mkDfsDir(SparkSession session, String dfsDir) { FileSystem fs = null; try { fs = FileSystem.get(session.sparkContext().hadoopConfiguration()); Path path = new Path(dfsDir); if(!fs.exists(path)) { fs.mkdirs(path); } } catch (IOException e) { e.printStackTrace(); } finally { try { if (null != fs) {fs.close();} } catch (IOException e) { e.printStackTrace(); } } } /** * 容错Driver * @param checkPointDir * @param second * @param namesrv * @param topic * @param group * @return */ static JavaStreamingContext createJavaSparkStreamingContext(String checkPointDir, int second, String namesrv, String topic, String group) { try { SparkConf conf = new SparkConf() //==== Enable Back Pressure .set("spark.streaming.backpressure.enabled", "true")//启用被压机制 .set("spark.streaming.backpressure.initialRate", "50000")//初始接收数据条数，如该值为空时使用spark.streaming.backpressure.initialRate为默认值 .set("spark.streaming.receiver.maxRate", "100")//每秒接收器可接收的最大记录数 //==== Enable Dynamic Resource Allocation .set("spark.dynamicAllocation.enabled", "false")//禁用spark动态资源分配 .set("spark.streaming.dynamicAllocation.enabled", "true")//启用SparkStreaming动态资源分配，该配置和spark动态资源分配存在冲突，只能使用一个 .set("spark.streaming.dynamicAllocation.minExecutors", "2")//启用SparkStreaming动态资源分配后的给应用使用的最小executor数 .set("spark.streaming.dynamicAllocation.maxExecutors", "3")//启用SparkStreaming动态资源分配后的给应用使用的最大executor数 //==== Spark Streaming Parallelism and WAL .set("spark.streaming.concurrentJobs", "1")//并行job数量，默认1 .set("spark.streaming.blockInterval", "5000")//SparkStreaming接收器接收数据后5000毫秒生成block，默认200毫秒 .set("spark.streaming.receiver.writeAheadLog.enable", "true")//开启SparkStreaming接收器的WAL来确保接收器实现至少一次的容错语义 .set("spark.streaming.driver.writeAheadLog.allowBatching", "true")//driver端WAL .set("spark.streaming.driver.writeAheadLog.batchingTimeout", "15000") //==== Hive on Spark .set("hive.execution.engine", "spark")//设置hive引擎为spark，hdp-2.6.1.0默认支持tez、mr，可通过应用级别配置使用Hive on Spark .set("hive.enable.spark.execution.engine", "true")//启用Hive on Spark .set("spark.driver.extraJavaOptions", "-Dhdp.version=2.6.1.0-129")//hdp-2.6.1.0中要求Hive on Spark必须指定driver的jvm参数 .set("spark.yarn.am.extraJavaOptions", "-Dhdp.version=2.6.1.0-129")//hdp-2.6.1.0中要求Hive on Spark必须指定ApplicationMaster的jvm参数 //==== Hive Merge Small Files .set("hive.metastore.uris", "thrift://bd001:9083")//hive ThriftServer .set("hive.merge.sparkfiles", "true")//合并spark小文件 .set("hive.merge.mapfiles", "true")//在只有map任务的作业结束时合并小文件。 .set("hive.merge.mapredfiles", "true")//在mapreduce作业结束时合并小文件。 .set("hive.merge.size.per.task", "268435456")//作业结束时合并文件的大小。 .set("hive.merge.smallfiles.avgsize", "100000")//当作业的平均输出文件大小小于此数量时，Hive将启动另一个map-reduce作业，以将输出文件合并为更大的文件。如果hive.merge.mapfiles为true，则仅对仅map作业执行此操作;对于hive.merge.mapredfiles为true，仅对map-reduce作业执行此操作。 //==== Spark SQL Optimizer .set("spark.sql.warehouse.dir", "hdfs://bd001:8020/apps/hive/warehouse")//SparkSQL依赖的hive仓库地址 .set("spark.sql.files.maxPartitionBytes", "134217728")//SparkSQL读取文件数据时打包到一个分区的最大字节数 .set("spark.sql.files.openCostInBytes", "134217728")//当SparkSQL读取的文件中有大量小文件时，小于该值的文件将被合并处理，默认4M，此处设置为128M .set("spark.sql.shuffle.partitions", "600")//SparkSQL运行shuffle的并行度 .set("spark.sql.autoBroadcastJoinThreshold", "67108864")//设置为64M，执行join时自动广播小于该值的表，默认10M //==== Spark Core Configure .set("spark.rdd.compress","true")//开启rdd压缩以节省内存 .set("spark.default.parallelism", "600")//并行任务数 .set("spark.rpc.askTimeout", "300")//spark rpc超时时间 .set("spark.eventLog.enabled", "true")//开启eventLog //==== Application Configure .set("spark.app.name", appName)//Spark Application名称 .set("spark.master", "yarn")//运行模式为Spark on YARN .set("spark.deploy.mode", "cluster")//部署模式为yarn-cluster .set("spark.driver.memory", "4g")//driver内存4g .set("spark.driver.cores", "1")//driver计算vcore数量为1 .set("spark.executor.memory", "5g")//executor内存为4g .set("spark.executor.heartbeatInterval", "20000")//executor心跳间隔20秒，默认10秒 .set("spark.yarn.archive", "hdfs://bd001:8020/hdp/apps/2.6.1.0-129/spark2")//spark依赖jar存档到hdfs指定位置 .set("spark.executor.extraJavaOptions", "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UseConcMarkSweepGC")//打印GC详情和耗时 .set("spark.jars", "/usr/hdp/2.6.1.0-129/sqoop/lib/mysql-connector-java.jar")//如果使用了数据库驱动，则通过此配置即可 //==== Serialized Configure .set("spark.kryoserializer.buffer", "512k")//默认64k，设置为256k .set("spark.kryoserializer.buffer.max", "256m")//默认64m，设置为256m .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")//使用kryo序列化库 .registerKryoClasses(new Class[]{HashMap.class, BCD.class}) ; //构建MQ配置 @SuppressWarnings("serial") HashMap<String, String> mqConf = new HashMap<String, String>() {{ put(RocketMQConfig.NAME_SERVER_ADDR, namesrv); put(RocketMQConfig.CONSUMER_TOPIC, topic); put(RocketMQConfig.CONSUMER_GROUP, group); }}; JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(second)); jssc.checkpoint(checkPointDir); jssc.remember(Durations.minutes(1440)); //接収RocketMQ账单 JavaReceiverInputDStream<BCD> billListRDD = jssc.receiverStream(new RocketMQReceiver(mqConf, StorageLevels.MEMORY_AND_DISK_SER)); billListRDD .foreachRDD((rdd, time) -> { boolean isEmpty = rdd.partitions().isEmpty(); if (!isEmpty) { printStream(rdd, time); long start = System.currentTimeMillis(); String curTime = getCurrentDate(start).intern(); String dfsDir = (BASE_PATH+"sdt="+curTime+"/").intern(); //账单计数器 //LongAccumulator billAccumulator = BillAccumulator.getInstance(JavaSparkContext.fromSparkContext(rdd.context()), appName); //billAccumulator.add(rdd.count()); //初始化SparkSession SparkSession session = HiveSession.getInstance(conf); //维表等广播 BroadcastDIM dim = BroadcastWrapper.getInstance(JavaSparkContext.fromSparkContext(rdd.context())).getValue(); SaleAnalysisService service = dim.getService(); Dataset<Row> shop = dim.getShop(); Dataset<Row> type = dim.getType(); Dataset<Row> ruian = dim.getRuian(); String mallIdStr = dim.getMallIdStr(); Set<String> areaSet=dim.getAreaSet(); Map<String, Set<String>> areaMall=dim.getAreaMall(); Set<String> mallSet=dim.getMallSet(); Set<String> typeSet=dim.getTypeSet(); //如果时间为00:00:00时则认为是新的一天，更新广播数据 if ((curTime+"000000").equals(FORMATTER_YMDHMS.get().format(new Date(time.milliseconds())))) { BroadcastWrapper.update(session, JavaSparkContext.fromSparkContext(rdd.context()));//更新dim表数据 mkDfsDir(session, dfsDir);//初始化dfsDir } //先持久化本次接收到的账单（写入当日） session.createDataFrame(rdd, BCD.class) .filter("sdt = "+curTime) .write() .mode("append") .parquet(dfsDir); //再读取全量账单（读取当日） Dataset<Row> bills = session.read().parquet(dfsDir) .filter("shopId in ("+mallIdStr+")") .dropDuplicates("billId") //.coalesce(1) .cache(); //计算大屏指标 System.out.println("==== 计算指标：时间条件："+curTime+" ===="); service.totalSale(bills); service.totalRefund(bills); service.peakTime(bills); service.areaSaleTrendForAll(bills, ruian,areaSet, curTime); service.projectContrast(bills, ruian,areaMall); service.saleForShopTop10(bills, shop, ruian); service.projectTypeSaleContrast(bills, shop, type, ruian,areaSet,mallSet,typeSet); long end = System.currentTimeMillis(); System.out.println("==== 计算指标：耗时："+(end-start)+"/ms ===="); bills.unpersist(); //每天最多存6个文件 if (bills.inputFiles().length > 6) { mergeSmallFiles(bills, session, dfsDir); } } else {//如果SparkStreaming接收的Batch为空，则不做处理 System.out.println("==== rdd is null! "); } }); return jssc; } catch (Exception e) { e.printStackTrace(); } return null; } } /** * 广播DIM相关数据 * @author mengyao * */ class BroadcastWrapper { private static volatile Broadcast<BroadcastDIM> instance = null; public static Broadcast<BroadcastDIM> getInstance(JavaSparkContext jsc) { if (instance == null) { synchronized (BroadcastWrapper.class) { if (instance == null) { SparkSession session = HiveSession.getInstance(jsc.getConf()); BroadcastDIM dim = new BroadcastDIM(session); dim.assign(); instance = jsc.broadcast(dim); } } } return instance; } /** * 每日更新数据 * @param batchTime batch发生时间 * @param dayES 每日开始时间 */ public static void update(SparkSession session, JavaSparkContext jsc) { BroadcastDIM dim = instance.getValue(); if (null!=dim) { dim.assign(); jsc.broadcast(dim); } } } class BroadcastDIM { private SparkSession session; private SaleAnalysisService service = new SaleAnalysisService(); private Dataset<Row> shop; private Dataset<Row> type; private Dataset<Row> ruian; private String mallIdStr; private List<RuianMall> ruianList ; private Set<String> areaSet; private Set<String> typeSet; private Set<String> mallSet; private Map<String, Set<String>> areaMall; public BroadcastDIM(SparkSession session) { this.session = session; } public void assign() { ruian = session.sql("select id,item,name,area_en,area_cn,channel,mid,rmid from tbl_dim_ruian").cache(); Row[] ruianRows = (Row[])ruian.collect(); setRuianList(ruianRows); setAreaSet(); setMallIdStr(); setRuianMallAll(); setMallSet(); shop = session.sql("select id,shop_entity_id,shop_entity_name,shop_id,shop_entity_type_root,shop_entity_type_leaf,bill_match_mode,leasing_model,shop_entity_status,open_time,close_time,open_time_list,close_time_list,monite_begin_time_list,monite_end_time_list,marketing_copywriter,marketing_image,font_style,font_size,contract_area,province,city,area,billhint,is_del,brand,brand_code,classify,in_aera,storey,leasing_resource,shop_entity_source,create_time,c_time_stamp,shop_entity_img,business_area_id,source,status,logo,door_head_photo,business_license,certificate,coordinates,consume_per,brand_name,brand_log,alipay,process " + "from tbl_ods_shop where shop_id in ("+mallIdStr+")").cache(); type = session.sql("select * from tbl_ods_type").cache(); setRuianType(); } public void setRuianList(Row[] rows) { if(rows.length>0){ ruianList=new ArrayList<>(); for(Row row:rows){ RuianMall rm=new RuianMall(); if(!row.isNullAt(0)){//id rm.setId(row.getInt(0)); } if(!row.isNullAt(1)){//item rm.setItem(row.getString(1)); } if(!row.isNullAt(2)){//name rm.setName(row.getString(2)); } if(!row.isNullAt(3)){//area_en rm.setAreaEn(row.getString(3)); } if(!row.isNullAt(4)){//area_cn rm.setArenCn(row.getString(4)); } if(!row.isNullAt(5)){//channel rm.setChannel(row.getString(5)); } if(!row.isNullAt(6)){//mid rm.setMid(row.getInt(6)); } if(!row.isNullAt(7)){//rmid rm.setRmid(row.getString(7)); } ruianList.add(rm); } } } /** * 提取rmid 拼接成字符串 * @param rows */ public void setMallIdStr() { if(ruianList.size()>0){ StringBuilder sbStr=new StringBuilder(); for(RuianMall row : ruianList){ sbStr.append("'").append(row.getRmid()).append("',"); } String tmpValue = sbStr.toString(); mallIdStr=tmpValue.substring(0, tmpValue.length()-1); } } /** * 提取非空唯一中文区域名称 * @return */ public void setAreaSet() { if(ruianList.size()>0){ areaSet=new java.util.HashSet<>(); for(RuianMall row:ruianList){ areaSet.add(row.getArenCn()); } } } /** * 提取瑞安mall 14个机构中文名 * * @return * Map<String,Set<String>> * key:areaCn value : Set<mallName> */ public void setRuianMallAll( ){ areaMall=new HashMap<>(); ruianList.forEach(rm->{ if(areaMall.containsKey(rm.getArenCn())){//存在，更新mall的列表 areaMall.get(rm.getArenCn()).add(rm.getName()); }else{//新的区域，新的mall Set<String> mallSet=new HashSet<>(); mallSet.add(rm.getName()); areaMall.put(rm.getArenCn(),mallSet); } }); } /** * 提取瑞安mall 14个机构中文名 * * @return */ public void setMallSet(){ mallSet=new java.util.HashSet<>(); ruianList.forEach(rm->{ if(!mallSet.contains(rm.getName())){//存在，更新mall的列表 mallSet.add(rm.getName()); } }); } /** * 性能、性能、性能 考虑，先写死 * 如果用账单、店铺、业态关联查询，效率会很慢 * * @return */ public void setRuianType(){ //TODO 目前写死，需要改 typeSet=new HashSet<>(); typeSet.add("服务"); typeSet.add("零售"); typeSet.add("娱乐"); typeSet.add("主力店"); typeSet.add("餐饮"); typeSet.add("其它"); } public SaleAnalysisService getService() { return service; } public void setService(SaleAnalysisService service) { this.service = service; } public Dataset<Row> getShop() { return shop; } public void setShop(Dataset<Row> shop) { this.shop = shop; } public Dataset<Row> getType() { return type; } public void setType(Dataset<Row> type) { this.type = type; } public Dataset<Row> getRuian() { return ruian; } public void setRuian(Dataset<Row> ruian) { this.ruian = ruian; } public String getMallIdStr() { return mallIdStr; } public void setMallIdStr(String mallIdStr) { this.mallIdStr = mallIdStr; } public List<RuianMall> getRuianList() { return ruianList; } public Set<String> getAreaSet() { return this.areaSet; } public Map<String, Set<String>> getAreaMall() { return this.areaMall; } public Set<String> getMallSet() { return this.mallSet; } public Set<String> getTypeSet() { return this.typeSet; } } /** * 账单累加器 * @author mengyao * */ class BillAccumulator { private static volatile LongAccumulator instance = null; public static LongAccumulator getInstance(JavaSparkContext jsc, String name) { if (instance == null) { synchronized (BillAccumulator.class) { if (instance == null) { instance = jsc.sc().longAccumulator(name); } } } return instance; } } /** * SparkSQL的Hive数据源 * @author mengyao * */ class HiveSession { private static transient SparkSession instance = null; public static SparkSession getInstance(SparkConf conf) { if (instance == null) { instance = SparkSession.builder() .config(conf) .enableHiveSupport() .getOrCreate(); } return instance; } }
4.4、账单实体类
package com.mengyao.graph.etl.apps.commons.beans.ods.fmt; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; import com.mengyao.graph.etl.apps.commons.beans.ods.BillInfo; import com.mengyao.graph.etl.apps.commons.beans.ods.DiscountDetailsInfo; import com.mengyao.graph.etl.apps.commons.beans.ods.GoodsDetailInfo; import com.mengyao.graph.etl.apps.commons.beans.ods.SettlementWayInfo; import com.mengyao.utils.DateTimeUtils; /** * 账单信息 gag_bill.bill_info * * @author jmb * @update 2018-12-13 for mengyao */ public class BillInfoFmt implements Serializable { private static final long serialVersionUID = 1L; /** * 预结单 */ public static final String BILLTYPE_JUJIEDAN = "2"; /** * 结账单 */ public static final String BILLTYPE_JIEZHANGDAN = "1"; /** * 日结账单 */ public static final String BILLTYPE_RIJIEDAN = "3"; /** * 点菜单 */ public static final String BILLTYPE_DIANCAIDAN = "7"; /** * 账单编号（系统产生），UUID */ private String id; /** * 账单编号（系统产生），HBase主键 */ private String rowKey; /** * 商家编号（系统产生） */ private String shopId; /** * 商家名称（系统产生） */ private String shopName; /** * 实体店编号（系统产生） */ private String shopEntityId; /** * 实体店名称（系统产生） */ private String shopEntityName; /** * 创建时间（系统产生） */ private String createTime; /** * 最后一次修改时间（系统产生） */ private String cTimeStamp; /** * 信息上传时间（以header中创建时间为准yyyyMMddHHmmss） */ private String hcTime; /** * 流水号(header中的流水号) */ private String hserial; /** * 修正账单数据的设备编号（如果没有修正，则与原始上传账单的采集终端编号相同） */ private String fixTerminal; /** * 采集终端编号（截获） */ private String terminalNumber; /** * 账单文件名称，唯一（截获），12位MAC地址+17位时间 */ private String billfileName; /** * 反扫支付时终端生成的终端流水号UUID（全球唯一）[先支付后打单必填] */ private String tsuuid; /** * 历史账单文件名称，记录合并过程中历次账单文件名称，全量，mongodb为Array */ private List<String> billfileNameHis; /** * 账单序号，不保证唯一（截获） */ private String billNo; /** * 截获时间（截获） */ private String interceptTime; /** * 店铺全名（截获） */ private String shopEntityFullName; /** * 店铺地址（截获） */ private String shopEntityAddress; /** * 电话（截获） */ private String telephone; /** * 售货员（截获） */ private String saler; /** * 收银台（截获） */ private String checkstand; /** * 收银员（截获） */ private String cashier; /** * 应收金额（截获） */ private Double receivableAmount; /** * 原始应收金额（截获） */ private Double defaultReceivableAmount; /** * 商品数量（截获） */ private Double totalNum; /** * 原始商品数量（截获） */ private Double defaultTotalNum; /** * 小票流水号（截获） */ private String billSerialNumber; /** * 总金额（截获） */ private Double totalFee; /** * 原始总金额（截获） */ private Double defaultTotalFee; /** * 实收金额（截获） */ private Double paidAmount; /** * 原始实收金额（截获） */ private Double defaultPaidAmount; /** * 折扣金额（截获） */ private Double discountAmount; /** * 原始折扣金额（截获） */ private Double defaultDiscountAmount; /** * 优惠金额（截获） */ private Double couponAmount; /** * 原始优惠金额（截获） */ private Double defaultCouponAmount; /** * 找零金额（截获） */ private Double changeAmount; /** * 原始找零金额（截获） */ private Double defaultChangeAmount; /** * 结算方式（支持多项）（截获）例：[{"a":5.0,"p":"现金"},{"a":10.01,"p":"书券"}],"a":结算金额，小数[截获，必填], * "p":"结算方式[截获，必填]" */ private List<String> settlementWay; /** * 销售时间（截获） */ private String saleTime; /** * 会员卡号（截获） */ private String memberCardNumber; /** * 累计消费（截获） */ private Double totalConsumption; /** * 原始累计消费（截获） */ private Double defaultTotalConsumption; /** * 网址（截获） */ private String website; /** * 小票图片（截获）,只存url */ private String billImage; /** * 商品详情（支持多项）（截获）[{"name":"可乐","itemserial":"PUMU00123","price":5.01,"totalnum":5.0,"totalprice":25.05},{"name":"金枪鱼","itemserial":"FOOD02012","price":10.55,"totalnum":1.5,"totalprice":15.83}] * ,"name":"商品名称[截获，选填]","itemserial":"条形码[截获，选填]","price":单价,小数[截获，选填],"totalnum":总数,小数[截获，选填],"totalprice":总价,小数[截获，选填] */ private List<String> goodsDetails; /** * 房间号（截获）（酒店特有） */ private String roomNo; /** * 入住姓名（截获）（酒店特有） */ private String checkinName; /** * 桌号（截获）（餐饮特有） */ private String deskNo; /** * 消费人数（截获）（一般用于餐饮） */ private Double consumeNum; /** * 原始消费人数（截获）（一般用于餐饮） */ private Double defaultConsumeNum; /** * 原始账单所有文本信息(截获) */ private String billText; /** * 入住时间(截获)(酒店特有) */ private String inTime; /** * 离店时间(截获)(钟点房特有) */ private String outTime; /** * 默认打印时间 */ private String defaultPrintDate; /** * 默认入住时间 */ private String defaultInTime; /** * 默认离店时间 */ private String defaultOutTime; /** * 上传类型 1：全单上传 2. 筛选账单上传 */ private String uploadType; /** * 除了上面截获外的自定义数据（截获），json串（Map<String, Object>）,json串中的key,value由采集终端自定义。 */ private Map<String, String> customRecord; /** * 账单类型1:结账单2:预结单 3:日结单 4:处方 5:预付押金单 6:退货单 7:点菜单 8:发票单 [选填] */ private String billType; /** * 账单修改方式 0:默认值 1:收银员重打 2:人工修改金额 3:自动重解析 */ private String modifyType = "0"; /** * 账单来源 1:设备采集 2:人工补录3、解析服务 4.第三方 */ private String billSource = "1"; /** * 服务端解析路径[选填，用于服务端解析分析问题] */ private String analyzPath; /** * 刷卡，钱包等匹配账单的key,格式BILL_MAC_金额_截获时间 */ private String billMatchKey; /** * 数据匹配支付结果等成功后,此字段保存支付结果等的主键 [选填] 匹配成功后为必填,理论上应该保证此值为全局唯一 */ private String matchId; /** * 默认销售时间; */ private String defaultSaleTime; /** * 客户名称[截获，选填] */ private String customerName; /** * 会员编号[截获，选填] */ private String membershipId; /** * 会员级别[截获，选填] */ private String memberLevels; /** * 宠物名称[截获，选填] */ private String petName; /** * 宠物编号[截获，选填] */ private String petNumber; /** * 负责人（医生）[截获，选填] */ private String principal; /** * 预交押金[截获，选填] */ private String deposit; /** * 打印时间yyyyMMddHHmmss[截获，选填，如果截获位数不够，后面补0] */ private String printDate; /** * 默认拦截时间 */ private String defaultInterceptTime; /** * 匹配模型 sk0001:先刷卡后打单,单次刷卡; dd0001:先打单后刷卡,单次刷卡 */ private String printMatchType; /** * 账单合并时使用,唯一 */ private String billMergeKey; /** * 存重打单的应收金额 */ private Double modifyAmount; /** * 存重打单的应收金额List */ private List<Double> modifyAmountList; /** * 存重打单的应收金额的销售时间List */ private List<String> modifyAmountSaleTimeList; /** * 存重打单的应收金额的截获时间List */ private List<String> modifyAmountInterceptTimeList; /** * 唯一标识[选填]（目前可用来做积分标识使用） */ private String uniqueId; /** * 历史唯一标识，记录合并过程中历次唯一标识，全量，mongodb为Array（目前可用来做积分标识使用） */ private List<String> uniqueIdHis; /** * 是否追加二维码 1：是 2：否 [必填] */ private String ifqrcode; /** * 优惠券券码 */ private String couponNum; /** * ERP会员 */ private String erpMemberCard; /** * 凭证类型 11:水单（商户pos、ERP打印）12:收银凭证（大pos打印） 13:支付凭证（签购单）99：类型未知 [选填，默认写99，表示未知] */ private String voucherType; /** * 小票二维码[选填] */ private String qrCode; /** * 积分标识 0：本次积分字段没找到或没有配置 1：有本次积分字段[选填] */ private String integralmark; /** * 本次积分[选填] */ private String thisintegral; /** * 备注[选填] */ private String remarks; /** * 账单子类型，庖丁用于配置文件分类 */ private String templateName; /** * 购货方名称[截获，选填] */ private String custName; /** * 购货方税号[截获，选填] */ private String custTaxNo; /** * 购货方地址、电话[截获，选填] */ private String custAdress; /** * 购货方银行及账号[截获，选填] */ private String custBankAccount; /** * 第三方订单号(第三方系统唯一) */ private String thirdPartyOrderNo; /** * 充值卡消费金额[截获，选填] */ private Double rechargeableCardConsumeAmount; /** * 原始充值卡消费金额[截获，选填] */ private Double defaultRechargeableCardConsumeAmount; /** * 实付金额[截获，选填] */ private Double outOfPocketAmount; /** * 原始实付金额[截获，选填] */ private Double defaultOutOfPocketAmount; /** * 充值金额[截获，选填] */ private Double rechargeAmount; /** * 原始充值金额[截获，选填] */ private Double defaultRechargeAmount; /** * 会员价[截获，选填] */ private Double memberPrice; /** * 原始会员价[截获，选填] */ private Double defaultMemberPrice; /** * 会员折扣率[截获，选填] */ private Double memberDiscountrate; /** * 原始会员折扣率[截获，选填] */ private Double defaultMemberDiscountrate; /** * 会员累计消费[截获，选填] */ private Double memberTotalConsumption; /** * 原始会员累计消费[截获，选填] */ private Double defaultMemberTotalConsumption; /** * 外卖单 1:外卖单 2:非外卖单[截获，选填] */ private String takeout; /** * 优惠商品详情（支持多项）（截获），{"name":"可乐","price":5.01} */ private List<String> discountDetails; public String getThirdPartyOrderNo() { return thirdPartyOrderNo; } public void setThirdPartyOrderNo(String thirdPartyOrderNo) { this.thirdPartyOrderNo = thirdPartyOrderNo; } public String getRemarks() { return remarks; } public void setRemarks(String remarks) { this.remarks = remarks; } public String getTemplateName() { return templateName; } public void setTemplateName(String templateName) { this.templateName = templateName; } public String getDefaultPrintDate() { return this.defaultPrintDate; } public void setDefaultPrintDate(String defaultPrintDate) { this.defaultPrintDate = defaultPrintDate; } public String getDefaultInTime() { return this.defaultInTime; } public void setDefaultInTime(String defaultInTime) { this.defaultInTime = defaultInTime; } public String getDefaultOutTime() { return this.defaultOutTime; } public void setDefaultOutTime(String defaultOutTime) { this.defaultOutTime = defaultOutTime; } public String getCouponNum() { return this.couponNum; } public void setCouponNum(String couponNum) { this.couponNum = couponNum; } public String getErpMemberCard() { return this.erpMemberCard; } public void setErpMemberCard(String erpMemberCard) { this.erpMemberCard = erpMemberCard; } public String getVoucherType() { return this.voucherType; } public void setVoucherType(String voucherType) { this.voucherType = voucherType; } public String getAnalyzPath() { return this.analyzPath; } public void setAnalyzPath(String analyzPath) { this.analyzPath = analyzPath; } public List<Double> getModifyAmountList() { return this.modifyAmountList; } public String getUniqueId() { return this.uniqueId; } public void setUniqueId(String uniqueId) { this.uniqueId = uniqueId; } public String getIfqrcode() { return this.ifqrcode; } public void setIfqrcode(String ifqrcode) { this.ifqrcode = ifqrcode; } public void setModifyAmountList(List<Double> modifyAmountList) { this.modifyAmountList = modifyAmountList; } public Double getModifyAmount() { return this.modifyAmount; } public void setModifyAmount(Double modifyAmount) { this.modifyAmount = modifyAmount; } public String getDefaultSaleTime() { return this.defaultSaleTime; } public void setDefaultSaleTime(String defaultSaleTime) { this.defaultSaleTime = defaultSaleTime; } public String getBillMergeKey() { return this.billMergeKey; } public void setBillMergeKey(String billMergeKey) { this.billMergeKey = billMergeKey; } public String getCustName() { return custName; } public void setCustName(String custName) { this.custName = custName; } public String getCustTaxNo() { return custTaxNo; } public void setCustTaxNo(String custTaxNo) { this.custTaxNo = custTaxNo; } public String getCustAdress() { return custAdress; } public void setCustAdress(String custAdress) { this.custAdress = custAdress; } public String getCustBankAccount() { return custBankAccount; } public void setCustBankAccount(String custBankAccount) { this.custBankAccount = custBankAccount; } public List<String> getUniqueIdHis() { return uniqueIdHis; } public void setUniqueIdHis(List<String> uniqueIdHis) { this.uniqueIdHis = uniqueIdHis; } public Double getRechargeableCardConsumeAmount() { return rechargeableCardConsumeAmount; } public void setRechargeableCardConsumeAmount(Double rechargeableCardConsumeAmount) { this.rechargeableCardConsumeAmount = rechargeableCardConsumeAmount; } public Double getDefaultRechargeableCardConsumeAmount() { return defaultRechargeableCardConsumeAmount; } public void setDefaultRechargeableCardConsumeAmount(Double defaultRechargeableCardConsumeAmount) { this.defaultRechargeableCardConsumeAmount = defaultRechargeableCardConsumeAmount; } public Double getOutOfPocketAmount() { return outOfPocketAmount; } public void setOutOfPocketAmount(Double outOfPocketAmount) { this.outOfPocketAmount = outOfPocketAmount; } public Double getDefaultOutOfPocketAmount() { return defaultOutOfPocketAmount; } public void setDefaultOutOfPocketAmount(Double defaultOutOfPocketAmount) { this.defaultOutOfPocketAmount = defaultOutOfPocketAmount; } public Double getRechargeAmount() { return rechargeAmount; } public void setRechargeAmount(Double rechargeAmount) { this.rechargeAmount = rechargeAmount; } public Double getDefaultRechargeAmount() { return defaultRechargeAmount; } public void setDefaultRechargeAmount(Double defaultRechargeAmount) { this.defaultRechargeAmount = defaultRechargeAmount; } public Double getMemberPrice() { return memberPrice; } public void setMemberPrice(Double memberPrice) { this.memberPrice = memberPrice; } public Double getDefaultMemberPrice() { return defaultMemberPrice; } public void setDefaultMemberPrice(Double defaultMemberPrice) { this.defaultMemberPrice = defaultMemberPrice; } public Double getMemberDiscountrate() { return memberDiscountrate; } public void setMemberDiscountrate(Double memberDiscountrate) { this.memberDiscountrate = memberDiscountrate; } public Double getDefaultMemberDiscountrate() { return defaultMemberDiscountrate; } public void setDefaultMemberDiscountrate(Double defaultMemberDiscountrate) { this.defaultMemberDiscountrate = defaultMemberDiscountrate; } public Double getMemberTotalConsumption() { return memberTotalConsumption; } public void setMemberTotalConsumption(Double memberTotalConsumption) { this.memberTotalConsumption = memberTotalConsumption; } public Double getDefaultMemberTotalConsumption() { return defaultMemberTotalConsumption; } public void setDefaultMemberTotalConsumption(Double defaultMemberTotalConsumption) { this.defaultMemberTotalConsumption = defaultMemberTotalConsumption; } public String getTakeout() { return takeout; } public void setTakeout(String takeout) { this.takeout = takeout; } public List<String> getDiscountDetails() { return discountDetails; } public void setDiscountDetails(List<String> discountDetails) { this.discountDetails = discountDetails; } public String getId() { return this.id; } public void setId(String id) { this.id = id; } public String getShopId() { return this.shopId; } public void setShopId(String shopId) { this.shopId = shopId; } public String getShopName() { return this.shopName; } public void setShopName(String shopName) { this.shopName = shopName; } public String getShopEntityId() { return this.shopEntityId; } public void setShopEntityId(String shopEntityId) { this.shopEntityId = shopEntityId; } public String getShopEntityName() { return this.shopEntityName; } public void setShopEntityName(String shopEntityName) { this.shopEntityName = shopEntityName; } public String getCreateTime() { return this.createTime; } public void setCreateTime(String createTime) { this.createTime = createTime; } public String getCTimeStamp() { return this.cTimeStamp; } public void setCTimeStamp(String cTimeStamp) { this.cTimeStamp = cTimeStamp; } public String getHcTime() { return this.hcTime; } public void setHcTime(String hcTime) { this.hcTime = hcTime; } public String getHserial() { return this.hserial; } public void setHserial(String hserial) { this.hserial = hserial; } public String getFixTerminal() { return this.fixTerminal; } public void setFixTerminal(String fixTerminal) { this.fixTerminal = fixTerminal; } public String getTerminalNumber() { return this.terminalNumber; } public void setTerminalNumber(String terminalNumber) { this.terminalNumber = terminalNumber; } public String getBillfileName() { return this.billfileName; } public void setBillfileName(String billfileName) { this.billfileName = billfileName; } public List<String> getBillfileNameHis() { return this.billfileNameHis; } public void setBillfileNameHis(List<String> billfileNameHis) { this.billfileNameHis = billfileNameHis; } public String getBillNo() { return this.billNo; } public void setBillNo(String billNo) { this.billNo = billNo; } public String getInterceptTime() { return this.interceptTime; } public void setInterceptTime(String interceptTime) { this.interceptTime = interceptTime; } public String getShopEntityFullName() { return this.shopEntityFullName; } public void setShopEntityFullName(String shopEntityFullName) { this.shopEntityFullName = shopEntityFullName; } public String getShopEntityAddress() { return this.shopEntityAddress; } public void setShopEntityAddress(String shopEntityAddress) { this.shopEntityAddress = shopEntityAddress; } public String getTelephone() { return this.telephone; } public void setTelephone(String telephone) { this.telephone = telephone; } public String getSaler() { return this.saler; } public void setSaler(String saler) { this.saler = saler; } public String getCheckstand() { return this.checkstand; } public void setCheckstand(String checkstand) { this.checkstand = checkstand; } public String getCashier() { return this.cashier; } public void setCashier(String cashier) { this.cashier = cashier; } public Double getReceivableAmount() { return this.receivableAmount; } public void setReceivableAmount(Double receivableAmount) { this.receivableAmount = receivableAmount; } public Double getTotalNum() { return this.totalNum; } public void setTotalNum(Double totalNum) { this.totalNum = totalNum; } public String getBillSerialNumber() { return this.billSerialNumber; } public void setBillSerialNumber(String billSerialNumber) { this.billSerialNumber = billSerialNumber; } public Double getTotalFee() { return this.totalFee; } public void setTotalFee(Double totalFee) { this.totalFee = totalFee; } public Double getPaidAmount() { return this.paidAmount; } public void setPaidAmount(Double paidAmount) { this.paidAmount = paidAmount; } public Double getDiscountAmount() { return this.discountAmount; } public void setDiscountAmount(Double discountAmount) { this.discountAmount = discountAmount; } public Double getCouponAmount() { return this.couponAmount; } public void setCouponAmount(Double couponAmount) { this.couponAmount = couponAmount; } public Double getChangeAmount() { return this.changeAmount; } public void setChangeAmount(Double changeAmount) { this.changeAmount = changeAmount; } public List<String> getSettlementWay() { return this.settlementWay; } public void setSettlementWay(List<String> settlementWay) { this.settlementWay = settlementWay; } public String getSaleTime() { return saleTime; } public void setSaleTime(String saleTime) { this.saleTime = saleTime; } public String getMemberCardNumber() { return this.memberCardNumber; } public void setMemberCardNumber(String memberCardNumber) { this.memberCardNumber = memberCardNumber; } public Double getTotalConsumption() { return this.totalConsumption; } public void setTotalConsumption(Double totalConsumption) { this.totalConsumption = totalConsumption; } public String getWebsite() { return this.website; } public void setWebsite(String website) { this.website = website; } public String getBillImage() { return this.billImage; } public void setBillImage(String billImage) { this.billImage = billImage; } public List<String> getGoodsDetails() { return this.goodsDetails; } public void setGoodsDetails(List<String> goodsDetails) { this.goodsDetails = goodsDetails; } public String getRoomNo() { return this.roomNo; } public void setRoomNo(String roomNo) { this.roomNo = roomNo; } public String getCheckinName() { return this.checkinName; } public void setCheckinName(String checkinName) { this.checkinName = checkinName; } public String getDeskNo() { return this.deskNo; } public void setDeskNo(String deskNo) { this.deskNo = deskNo; } public Double getConsumeNum() { return this.consumeNum; } public void setConsumeNum(Double consumeNum) { this.consumeNum = consumeNum; } public String getBillText() { return this.billText; } public void setBillText(String billText) { this.billText = billText; } public Map<String, String> getCustomRecord() { return this.customRecord; } public void setCustomRecord(Map<String, String> customRecord) { this.customRecord = customRecord; } public String getBillType() { return this.billType; } public void setBillType(String billType) { this.billType = billType; } public String getInTime() { return this.inTime; } public String getOutTime() { return this.outTime; } public void setInTime(String inTime) { this.inTime = inTime; } public void setOutTime(String outTime) { this.outTime = outTime; } public String getBillMatchKey() { return this.billMatchKey; } public void setBillMatchKey(String billMatchKey) { this.billMatchKey = billMatchKey; } public String getMatchId() { return this.matchId; } public void setMatchId(String matchId) { this.matchId = matchId; } public String getPrintMatchType() { return this.printMatchType; } public void setPrintMatchType(String printMatchType) { this.printMatchType = printMatchType; } public String getTsuuid() { return this.tsuuid; } public void setTsuuid(String tsuuid) { this.tsuuid = tsuuid; } public String getUploadType() { return this.uploadType; } public void setUploadType(String uploadType) { this.uploadType = uploadType; } public String getCustomerName() { return this.customerName; } public void setCustomerName(String customerName) { this.customerName = customerName; } public String getMembershipId() { return this.membershipId; } public void setMembershipId(String membershipId) { this.membershipId = membershipId; } public String getMemberLevels() { return this.memberLevels; } public void setMemberLevels(String memberLevels) { this.memberLevels = memberLevels; } public String getPetName() { return this.petName; } public void setPetName(String petName) { this.petName = petName; } public String getPetNumber() { return this.petNumber; } public void setPetNumber(String petNumber) { this.petNumber = petNumber; } public String getPrincipal() { return this.principal; } public void setPrincipal(String principal) { this.principal = principal; } public String getDeposit() { return this.deposit; } public void setDeposit(String deposit) { this.deposit = deposit; } public String getPrintDate() { return this.printDate; } public void setPrintDate(String printDate) { this.printDate = printDate; } public String getDefaultInterceptTime() { return this.defaultInterceptTime; } public void setDefaultInterceptTime(String defaultInterceptTime) { this.defaultInterceptTime = defaultInterceptTime; } public String getModifyType() { return this.modifyType; } public void setModifyType(String modifyType) { this.modifyType = modifyType; } public String getBillSource() { return this.billSource; } public void setBillSource(String billSource) { this.billSource = billSource; } public String getQrCode() { return qrCode; } public void setQrCode(String qrCode) { this.qrCode = qrCode; } public String getIntegralmark() { return integralmark; } public void setIntegralmark(String integralmark) { this.integralmark = integralmark; } public String getThisintegral() { return thisintegral; } public void setThisintegral(String thisintegral) { this.thisintegral = thisintegral; } public List<String> getModifyAmountSaleTimeList() { return modifyAmountSaleTimeList; } public void setModifyAmountSaleTimeList(List<String> modifyAmountSaleTimeList) { this.modifyAmountSaleTimeList = modifyAmountSaleTimeList; } public List<String> getModifyAmountInterceptTimeList() { return modifyAmountInterceptTimeList; } public void setModifyAmountInterceptTimeList(List<String> modifyAmountInterceptTimeList) { this.modifyAmountInterceptTimeList = modifyAmountInterceptTimeList; } public Double getDefaultReceivableAmount() { return defaultReceivableAmount; } public void setDefaultReceivableAmount(Double defaultReceivableAmount) { this.defaultReceivableAmount = defaultReceivableAmount; } public Double getDefaultTotalNum() { return defaultTotalNum; } public void setDefaultTotalNum(Double defaultTotalNum) { this.defaultTotalNum = defaultTotalNum; } public Double getDefaultTotalFee() { return defaultTotalFee; } public void setDefaultTotalFee(Double defaultTotalFee) { this.defaultTotalFee = defaultTotalFee; } public Double getDefaultPaidAmount() { return defaultPaidAmount; } public void setDefaultPaidAmount(Double defaultPaidAmount) { this.defaultPaidAmount = defaultPaidAmount; } public Double getDefaultDiscountAmount() { return defaultDiscountAmount; } public void setDefaultDiscountAmount(Double defaultDiscountAmount) { this.defaultDiscountAmount = defaultDiscountAmount; } public Double getDefaultCouponAmount() { return defaultCouponAmount; } public void setDefaultCouponAmount(Double defaultCouponAmount) { this.defaultCouponAmount = defaultCouponAmount; } public Double getDefaultChangeAmount() { return defaultChangeAmount; } public void setDefaultChangeAmount(Double defaultChangeAmount) { this.defaultChangeAmount = defaultChangeAmount; } public Double getDefaultTotalConsumption() { return defaultTotalConsumption; } public void setDefaultTotalConsumption(Double defaultTotalConsumption) { this.defaultTotalConsumption = defaultTotalConsumption; } public Double getDefaultConsumeNum() { return defaultConsumeNum; } public void setDefaultConsumeNum(Double defaultConsumeNum) { this.defaultConsumeNum = defaultConsumeNum; } public String getRowKey() { return rowKey; } public void setRowKey(String rowKey) { this.rowKey = rowKey; } @Override public String toString() { return id + "\t" + rowKey + "\t" + shopId + "\t" + shopName + "\t" + shopEntityId + "\t" + shopEntityName + "\t" + createTime + "\t" + cTimeStamp + "\t" + hcTime + "\t" + hserial + "\t" + fixTerminal + "\t" + terminalNumber + "\t" + billfileName + "\t" + tsuuid + "\t" + billfileNameHis + "\t" + billNo + "\t" + interceptTime + "\t" + shopEntityFullName + "\t" + shopEntityAddress + "\t" + telephone + "\t" + saler + "\t" + checkstand + "\t" + cashier + "\t" + receivableAmount + "\t" + defaultReceivableAmount + "\t" + totalNum + "\t" + defaultTotalNum + "\t" + billSerialNumber + "\t" + totalFee + "\t" + defaultTotalFee + "\t" + paidAmount + "\t" + defaultPaidAmount + "\t" + discountAmount + "\t" + defaultDiscountAmount + "\t" + couponAmount + "\t" + defaultCouponAmount + "\t" + changeAmount + "\t" + defaultChangeAmount + "\t" + settlementWay + "\t" + saleTime + "\t" + memberCardNumber + "\t" + totalConsumption + "\t" + defaultTotalConsumption + "\t" + website + "\t" + billImage + "\t" + goodsDetails + "\t" + roomNo + "\t" + checkinName + "\t" + deskNo + "\t" + consumeNum + "\t" + defaultConsumeNum + "\t" + billText + "\t" + inTime + "\t" + outTime + "\t" + defaultPrintDate + "\t" + defaultInTime + "\t" + defaultOutTime + "\t" + uploadType + "\t" + customRecord + "\t" + billType + "\t" + modifyType + "\t" + billSource + "\t" + analyzPath + "\t" + billMatchKey + "\t" + matchId + "\t" + defaultSaleTime + "\t" + customerName + "\t" + membershipId + "\t" + memberLevels + "\t" + petName + "\t" + petNumber + "\t" + principal + "\t" + deposit + "\t" + printDate + "\t" + defaultInterceptTime + "\t" + printMatchType + "\t" + billMergeKey + "\t" + modifyAmount + "\t" + modifyAmountList + "\t" + modifyAmountSaleTimeList + "\t" + modifyAmountInterceptTimeList + "\t" + uniqueId + "\t" + uniqueIdHis + "\t" + ifqrcode + "\t" + couponNum + "\t" + erpMemberCard + "\t" + voucherType + "\t" + qrCode + "\t" + integralmark + "\t" + thisintegral + "\t" + remarks + "\t" + templateName + "\t" + custName + "\t" + custTaxNo + "\t" + custAdress + "\t" + custBankAccount + "\t" + thirdPartyOrderNo + "\t" + rechargeableCardConsumeAmount + "\t" + defaultRechargeableCardConsumeAmount + "\t" + outOfPocketAmount + "\t" + defaultOutOfPocketAmount + "\t" + rechargeAmount + "\t" + defaultRechargeAmount + "\t" + memberPrice + "\t" + defaultMemberPrice + "\t" + memberDiscountrate + "\t" + defaultMemberDiscountrate + "\t" + memberTotalConsumption + "\t" + defaultMemberTotalConsumption + "\t" + takeout + "\t" + discountDetails; } public static BillInfoFmt cloneBill(BillInfo fromBean) { if (null == fromBean) { return null; } BillInfoFmt toBean = new BillInfoFmt(); toBean.setId(stringRemove(fromBean.getId())); toBean.setRowKey(stringRemove(fromBean.getRowKey())); toBean.setShopId(stringRemove(fromBean.getShopId())); toBean.setShopName(stringRemove(fromBean.getShopName())); toBean.setShopEntityId(stringRemove(fromBean.getShopEntityId())); toBean.setShopEntityName(stringRemove(fromBean.getShopEntityName())); toBean.setCreateTime(DateTimeUtils.getYmdhmsForNo(fromBean.getCreateTime())); toBean.setCTimeStamp(DateTimeUtils.getYmdhmsForNo(fromBean.getCTimeStamp())); toBean.setHcTime(stringRemove(fromBean.getHcTime())); toBean.setHserial(stringRemove(fromBean.getHserial())); toBean.setFixTerminal(stringRemove(fromBean.getFixTerminal())); toBean.setTerminalNumber(stringRemove(fromBean.getTerminalNumber())); toBean.setBillfileName(stringRemove(fromBean.getBillfileName())); toBean.setTsuuid(stringRemove(fromBean.getTsuuid())); toBean.setBillfileNameHis(fromBean.getBillfileNameHis()); toBean.setBillNo(stringRemove(fromBean.getBillNo())); toBean.setInterceptTime(DateTimeUtils.getYmdhmsForNo(fromBean.getInterceptTime())); toBean.setShopEntityFullName(stringRemove(fromBean.getShopEntityFullName())); toBean.setShopEntityAddress(stringRemove(fromBean.getShopEntityAddress())); toBean.setTelephone(stringRemove(fromBean.getTelephone())); toBean.setSaler(stringRemove(fromBean.getSaler())); toBean.setCheckstand(stringRemove(fromBean.getCheckstand())); toBean.setCashier(stringRemove(fromBean.getCashier())); toBean.setReceivableAmount(fromBean.getReceivableAmount()); toBean.setDefaultReceivableAmount(fromBean.getDefaultReceivableAmount()); toBean.setTotalNum(fromBean.getTotalNum()); toBean.setDefaultTotalNum(fromBean.getDefaultTotalNum()); toBean.setBillSerialNumber(stringRemove(fromBean.getBillSerialNumber())); toBean.setTotalFee(fromBean.getTotalFee()); toBean.setDefaultTotalFee(fromBean.getDefaultTotalFee()); toBean.setPaidAmount(fromBean.getPaidAmount()); toBean.setDefaultPaidAmount(fromBean.getDefaultPaidAmount()); toBean.setDiscountAmount(fromBean.getDiscountAmount()); toBean.setDefaultDiscountAmount(fromBean.getDefaultDiscountAmount()); toBean.setCouponAmount(fromBean.getCouponAmount()); toBean.setDefaultCouponAmount(fromBean.getDefaultCouponAmount()); toBean.setChangeAmount(fromBean.getChangeAmount()); toBean.setDefaultChangeAmount(fromBean.getDefaultChangeAmount()); if (null != fromBean.getSettlementWay()) { List<SettlementWayInfo> rawList = fromBean.getSettlementWay(); List<String> list = new ArrayList<>(); for (SettlementWayInfo raw : rawList) { list.add(raw.toString()); } toBean.setSettlementWay(list); } toBean.setSaleTime(fromBean.getSaleTime()); toBean.setMemberCardNumber(stringRemove(fromBean.getMemberCardNumber())); toBean.setTotalConsumption(fromBean.getTotalConsumption()); toBean.setDefaultTotalConsumption(fromBean.getDefaultTotalConsumption()); toBean.setWebsite(stringRemove(fromBean.getWebsite())); toBean.setBillImage(stringRemove(fromBean.getBillImage())); if (null != fromBean.getGoodsDetails()) { List<GoodsDetailInfo> rawList = fromBean.getGoodsDetails(); List<String> list = new ArrayList<>(); for (GoodsDetailInfo raw : rawList) { list.add(raw.toString()); } toBean.setGoodsDetails(list); } toBean.setRoomNo(stringRemove(fromBean.getRoomNo())); toBean.setCheckinName(stringRemove(fromBean.getCheckinName())); toBean.setDeskNo(stringRemove(fromBean.getDeskNo())); toBean.setConsumeNum(fromBean.getConsumeNum()); toBean.setDefaultConsumeNum(fromBean.getDefaultConsumeNum()); toBean.setBillText(stringRemove(fromBean.getBillText())); toBean.setInTime(DateTimeUtils.getYmdhmsForNo(fromBean.getInTime())); toBean.setOutTime(DateTimeUtils.getYmdhmsForNo(fromBean.getOutTime())); toBean.setDefaultPrintDate(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultPrintDate())); toBean.setDefaultInTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultInTime())); toBean.setDefaultOutTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultOutTime())); toBean.setUploadType(stringRemove(fromBean.getUploadType())); toBean.setCustomRecord(fromBean.getCustomRecord()); toBean.setBillType(stringRemove(fromBean.getBillType())); toBean.setModifyType(stringRemove(fromBean.getModifyType())); toBean.setBillSource(stringRemove(fromBean.getBillSource())); toBean.setAnalyzPath(stringRemove(fromBean.getAnalyzPath())); toBean.setBillMatchKey(stringRemove(fromBean.getBillMatchKey())); toBean.setMatchId(stringRemove(fromBean.getMatchId())); toBean.setDefaultSaleTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultSaleTime())); toBean.setCustomerName(stringRemove(fromBean.getCustomerName())); toBean.setMembershipId(stringRemove(fromBean.getMembershipId())); toBean.setMemberLevels(stringRemove(fromBean.getMemberLevels())); toBean.setPetName(stringRemove(fromBean.getPetName())); toBean.setPetNumber(stringRemove(fromBean.getPetNumber())); toBean.setPrincipal(stringRemove(fromBean.getPrincipal())); toBean.setDeposit(stringRemove(fromBean.getDeposit())); toBean.setPrintDate(DateTimeUtils.getYmdhmsForNo(fromBean.getPrintDate())); toBean.setDefaultInterceptTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultInterceptTime())); toBean.setPrintMatchType(stringRemove(fromBean.getPrintMatchType())); toBean.setBillMergeKey(stringRemove(fromBean.getBillMergeKey())); toBean.setModifyAmount(fromBean.getModifyAmount()); toBean.setModifyAmountList(fromBean.getModifyAmountList()); toBean.setModifyAmountSaleTimeList(DateTimeUtils.getYmdhmsForNo(fromBean.getModifyAmountSaleTimeList())); toBean.setModifyAmountInterceptTimeList( DateTimeUtils.getYmdhmsForNo(fromBean.getModifyAmountInterceptTimeList())); toBean.setUniqueId(stringRemove(fromBean.getUniqueId())); toBean.setUniqueIdHis(fromBean.getUniqueIdHis()); toBean.setIfqrcode(stringRemove(fromBean.getIfqrcode())); toBean.setCouponNum(stringRemove(fromBean.getCouponNum())); toBean.setErpMemberCard(stringRemove(fromBean.getErpMemberCard())); toBean.setVoucherType(stringRemove(fromBean.getVoucherType())); toBean.setQrCode(stringRemove(fromBean.getQrCode())); toBean.setIntegralmark(stringRemove(fromBean.getIntegralmark())); toBean.setThisintegral(stringRemove(fromBean.getThisintegral())); toBean.setRemarks(stringRemove(fromBean.getRemarks())); toBean.setTemplateName(stringRemove(fromBean.getTemplateName())); toBean.setCustName(stringRemove(fromBean.getCustName())); toBean.setCustName(stringRemove(fromBean.getCustTaxNo())); toBean.setCustAdress(stringRemove(fromBean.getCustAdress())); toBean.setCustBankAccount(stringRemove(fromBean.getCustBankAccount())); toBean.setThirdPartyOrderNo(stringRemove(fromBean.getThirdPartyOrderNo())); toBean.setRechargeableCardConsumeAmount(fromBean.getRechargeableCardConsumeAmount()); toBean.setDefaultRechargeableCardConsumeAmount(fromBean.getDefaultRechargeableCardConsumeAmount()); toBean.setOutOfPocketAmount(fromBean.getOutOfPocketAmount()); toBean.setDefaultOutOfPocketAmount(fromBean.getDefaultOutOfPocketAmount()); toBean.setRechargeAmount(fromBean.getRechargeAmount()); toBean.setDefaultRechargeAmount(fromBean.getDefaultRechargeAmount()); toBean.setMemberPrice(fromBean.getMemberPrice()); toBean.setDefaultMemberPrice(fromBean.getDefaultMemberPrice()); toBean.setMemberDiscountrate(fromBean.getMemberDiscountrate()); toBean.setDefaultMemberDiscountrate(fromBean.getDefaultMemberDiscountrate()); toBean.setMemberTotalConsumption(fromBean.getMemberTotalConsumption()); toBean.setDefaultMemberTotalConsumption(fromBean.getDefaultMemberTotalConsumption()); toBean.setTakeout(stringRemove(fromBean.getTakeout())); if (null != fromBean.getDiscountDetails()) { List<DiscountDetailsInfo> rawList = fromBean.getDiscountDetails(); List<String> list = new ArrayList<>(); for (DiscountDetailsInfo raw : rawList) { list.add(raw.toString()); } toBean.setDiscountDetails(list); } return toBean; } /** * 原始数据格式化处理，待完善 * * @param jsonStr * @return */ private static String stringRemove(String strVal) { if (StringUtils.isEmpty(strVal)) { return ""; } strVal = strVal.replace("\t", ""); strVal = strVal.replace("\n", ""); strVal = strVal.replace("\r", ""); return strVal; } }
4.5、BCD裁剪实体类
package com.mengyao.graph.etl.apps.dashboard.beans; import java.io.Serializable; /** * Bill Consumer Dashboard Bean * @author mengyao * */ public class BCD implements Serializable { private static final long serialVersionUID = 1749406742944513387L; private String billId; private double receivableAmount; private String saleTime; private String sdt;//yyyyMMdd private int hour; private String billType; private String shopId; private String shopEntityId; public BCD() { super(); } public BCD(String billId, double receivableAmount, String saleTime, String billType, String shopId, String shopEntityId) { super(); this.billId = billId; this.receivableAmount = receivableAmount; this.saleTime = saleTime; setDateTime(saleTime); this.billType = billType; this.shopId = shopId; this.shopEntityId = shopEntityId; } public String getBillId() { return billId; } public void setBillId(String billId) { this.billId = billId; } public double getReceivableAmount() { return receivableAmount; } public void setReceivableAmount(double receivableAmount) { this.receivableAmount = receivableAmount; } public String getSaleTime() { return saleTime; } public void setSaleTime(String saleTime) { this.saleTime = saleTime; setDateTime(saleTime); } public String getSdt() { return sdt; } public void setSdt(String sdt) { this.sdt = sdt; } public int getHour() { return hour; } public void setHour(int hour) { this.hour = hour; } public String getBillType() { return billType; } public void setBillType(String billType) { this.billType = billType; } public String getShopId() { return shopId; } public void setShopId(String shopId) { this.shopId = shopId; } public String getShopEntityId() { return shopEntityId; } public void setShopEntityId(String shopEntityId) { this.shopEntityId = shopEntityId; } private void setDateTime(String saleTime) { if (null!=saleTime&&saleTime.length()==17) { this.sdt = saleTime.substring(0, 8); this.hour = Integer.parseInt(saleTime.substring(8, 10)); } } @Override public String toString() { return billId + "\t" + receivableAmount + "\t" + saleTime + "\t" + sdt + "\t" + billType + "\t" + shopId + "\t" + shopEntityId; } }
4.6、销售分析业务实现类
package com.mengyao.graph.etl.apps.dashboard.service; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.functions; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.mengyao.graph.etl.apps.dashboard.beans.AreaProjSale; import com.mengyao.graph.etl.apps.dashboard.beans.AreaSaleTrendAll; import com.mengyao.graph.etl.apps.dashboard.beans.PeakTime; import com.mengyao.graph.etl.apps.dashboard.beans.ProjectTypeShopNumber; import com.mengyao.graph.etl.apps.dashboard.beans.ShopSaleRank; import com.mengyao.graph.etl.apps.dashboard.beans.TotalRefund; import com.mengyao.graph.etl.apps.dashboard.beans.TotalSale; import com.mengyao.graph.etl.apps.dashboard.beans.TotalSettlementBillNumber; import com.mengyao.utils.RedisUtil; /** * 大屏指标分析 * @author mengyao * */ public class SaleAnalysisService implements Serializable { private static final long serialVersionUID = 8289368096001689148L; //解决区域名称hash重复问题 private static final String SALT="aAb12"; /** * 每日重置大屏指标值 */ @Deprecated public void reset() { RedisUtil.setObject("dtsbn_6", "{\"结账单数\":{\"val\":7270}}\""); RedisUtil.setObject("dpt_7", "{\"高峰时段\":{\"val\":13}}\""); RedisUtil.setObject("dtr_5", "{\"退款金额\":{\"val\":7301.8}}\""); RedisUtil.setObject("dts_4", "{\"总销售额\":{\"val\":1300523.8000000005}}\""); RedisUtil.setObject("curDay", "20190227\""); RedisUtil.setObject("dastfa_2_20190227", "{\"各区域销售额发展趋势\":{\"重庆天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,9.5,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"创智天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,43.3,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"上海瑞虹\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"上海新天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,553.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]}}\""); RedisUtil.setObject("dasp_1", "{\"各区域销售额占比\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\""); RedisUtil.setObject("dpc_8", "{\"各项目销售额对比\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\""); RedisUtil.setObject("dpac_10", "{\"各项目单均消费\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\""); RedisUtil.setObject("dsfst_9", "{\"店铺销售排行\":[{\"pn\":\"新天地时尚购物中心\",\"sa\":152.0,\"sn\":\"GREYBOX COFFEE\"},{\"pn\":\"新天地广场\",\"sa\":114.75,\"sn\":\"Arabica\"},{\"pn\":\"湖滨道购物中心\",\"sa\":100.0,\"sn\":\"LOKAL\"},{\"pn\":\"上海新天地南里北里\",\"sa\":74.0,\"sn\":\"哈肯铺_手感烘焙\"},{\"pn\":\"壹方\",\"sa\":60.3,\"sn\":\"新一天便利\"},{\"pn\":\"新天地广场\",\"sa\":41.0,\"sn\":\"奈雪的茶\"},{\"pn\":\"新天地广场\",\"sa\":39.0,\"sn\":\"蒲石小点\"},{\"pn\":\"新天地广场\",\"sa\":18.0,\"sn\":\"Fresh_Every_Day\"},{\"pn\":\"瑞虹天地星星堂\",\"sa\":16.0,\"sn\":\"老盛昌\"},{\"pn\":\"新天地广场\",\"sa\":15.0,\"sn\":\"多几谷\"}]}\""); RedisUtil.setObject("dptsc_11", "{\"各项目业态销售额对比\":{\"上海新天地南里北里\":[{\"n\":\"餐饮\",\"sa\":74.0,\"sn\":3}],\"新天地广场\":[{\"n\":\"餐饮\",\"sa\":227.75,\"sn\":10}],\"重庆天地\":[{\"n\":\"其它\",\"sa\":9.5,\"sn\":0}],\"湖滨道购物中心\":[{\"n\":\"餐饮\",\"sa\":100.0,\"sn\":2}],\"壹方\":[{\"n\":\"餐饮\",\"sa\":43.3,\"sn\":3}],\"瑞虹天地星星堂\":[{\"n\":\"餐饮\",\"sa\":16.0,\"sn\":1}],\"新天地时尚购物中心\":[{\"n\":\"餐饮\",\"sa\":152.0,\"sn\":4}]}}\""); RedisUtil.setObject("dpsn_12", "{\"各项目店铺数量\":{\"上海新天地南里北里\":[{\"n\":\"餐饮\",\"sa\":74.0,\"sn\":3}],\"新天地广场\":[{\"n\":\"餐饮\",\"sa\":227.75,\"sn\":10}],\"重庆天地\":[{\"n\":\"其它\",\"sa\":9.5,\"sn\":0}],\"湖滨道购物中心\":[{\"n\":\"餐饮\",\"sa\":100.0,\"sn\":2}],\"壹方\":[{\"n\":\"餐饮\",\"sa\":43.3,\"sn\":3}],\"瑞虹天地星星堂\":[{\"n\":\"餐饮\",\"sa\":16.0,\"sn\":1}],\"新天地时尚购物中心\":[{\"n\":\"餐饮\",\"sa\":152.0,\"sn\":4}]}}\""); } /** * 总销售额 * @param ds */ public void totalSale(Dataset<Row> bill) { Row[] rows = (Row[])bill .filter("billType=1 and receivableAmount>=0") .agg(functions.sum(new Column("receivableAmount")).alias("totalSale"), functions.count("receivableAmount")) .head(1); Map<String, TotalSale> map = new HashMap<String, TotalSale>(); double totalSale=0D; if(rows.length>0){ Row row = rows[0]; if (!row.isNullAt(0)) { //销售额 totalSale=row.getDouble(0); } if (!row.isNullAt(1)) { //结账单数 totalSettlementBillNumber(new TotalSettlementBillNumber(row.getLong(1))); } } map.put("总销售额", new TotalSale(totalSale)); String str = JSONObject.toJSONString(map); System.out.println("====####--totalSale##" + str); // 将总销售额存入redis RedisUtil.setObject("dts_4", str); } /** * 退款金额 * @param ds */ public void totalRefund(Dataset<Row> bill) { Row[] rows = (Row[]) bill .filter("((billType=6) or (billType=1 and receivableAmount<0))") .agg(functions.sum(functions.abs(new Column("receivableAmount"))).alias("totalSale")) .head(1); Map<String, TotalRefund> map=new HashMap<>(); map.put("退款金额", new TotalRefund(0)); if(rows.length>0){ Row row = rows[0]; if (!row.isNullAt(0)) { map.put("退款金额", new TotalRefund(row.getDouble(0))); } } String str=JSONObject.toJSONString(map); System.out.println("====####--totalRefund##"+str); //将退款金额存入redis RedisUtil.setObject("dtr_5",str); } /** * 高峰时段 * 1、时段：小时； * 2、高峰：当日每小时结账单数累计最大； * 3、高峰时段：所有mall累计每小时结账单数； * @param ds * @return {"高峰时段":{"val":12}} */ public void peakTime(Dataset<Row> bill) { //账单表本身有mallid(shopId) 因此无需关联店铺表 Map<String, PeakTime> map=new HashMap<>(); map.put("高峰时段", new PeakTime(8)); Row[] rows = (Row[])bill .filter("billType=1") .groupBy("hour") .agg(functions.sum("receivableAmount").alias("totalSale")) .orderBy(new Column("totalSale").desc()) .limit(1) .head(1); if(rows.length>0){ Row row = rows[0]; if (!row.isNullAt(0)) { map.put("高峰时段", new PeakTime(row.getAs(0))); } } String str=JSONObject.toJSONString(map); System.out.println("====####--peakTime##"+str); //将高峰时段放入redis RedisUtil.setObject("dpt_7", str); } /** * 各区域销售发展趋势-多个区域（每个区域下有多个mall）当日0点~24点的累计销售额 * @param bill * @param ruian * @param curDay */ public void areaSaleTrendForAll(Dataset<Row> bill, Dataset<Row> ruian,Set<String> areaSet, String curDay) { Row[] rows = (Row[])bill .filter("billType=1 and receivableAmount>0") .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter") .groupBy("area_cn", "hour") .agg(functions.sum("receivableAmount").alias("areaDayHourSale")) .orderBy("areaDayHourSale") .select("area_cn","areaDayHourSale","hour") .collect(); // Map<String, Map<String,Map<Integer, Double>>> map = new HashMap<>(); Map<String, Map<String,Collection<Double>>> map = new HashMap<>(); Map<String,AreaSaleTrendAll> maps=new HashMap<>(); if(rows.length>0){ for (Row row : rows) { String areaCn=null; double areaDayHourSale=0D; int saleHour=0; if(!row.isNullAt(0)){ areaCn=row.getString(0); } if(!row.isNullAt(1)){ areaDayHourSale=row.getDouble(1); } if(!row.isNullAt(2)){ saleHour=row.getInt(2); } if(maps.containsKey(areaCn+SALT)){ AreaSaleTrendAll ast=maps.get(areaCn+SALT); ast.getVals().put(saleHour, areaDayHourSale); }else{ HashMap<Integer,Double> vals=new HashMap<Integer,Double>(); vals.put(saleHour, areaDayHourSale); maps.put(areaCn+SALT, new AreaSaleTrendAll(areaCn, saleHour, areaDayHourSale)); } } } //填充没有销售额的区域记录，使数据更加完整。 areaSet.forEach(areaCn->{ if(!maps.containsKey(areaCn+SALT)){ maps.put(areaCn+SALT, new AreaSaleTrendAll(areaCn)); } }); System.out.println("==####========填充hou====begin====================="); for (String key:maps.keySet()) { System.out.println("Key:"+key); } System.out.println("==####========填充hou=====end===================="); // Map<String,Map<Integer, Double>> rs=new HashMap<>(); // for(AreaSaleTrendAll asta:maps.values()){ // rs.put(asta.getAreaCn(), asta.getVals()); // } Map<String,Collection<Double>> rs=new HashMap<>(); for(AreaSaleTrendAll asta:maps.values()){ rs.put(asta.getAreaCn(), asta.getVals().values()); } map.put("各区域销售额发展趋势", rs); //转成json 字符串 String str=JSONObject.toJSONString(map); System.out.println("====####--areaSaleTrendForAll##"+str); //放入redis RedisUtil.setObject("dastfa_2_"+curDay, str); //维护redis中最新日期 if(!RedisUtil.existsObject("curDay")){//如果为空，说明是第一次运行，直接将当前日期设置到redis中 RedisUtil.setObject("curDay", curDay); }else{ String curDayRedis=(String) RedisUtil.getObject("curDay"); if((curDayRedis.compareTo(curDay))<0){//说明当前日期大于redis中的日期，更新 RedisUtil.setObject("curDay", curDay); } } } /** * 各项目销售额对比 * 1、各项目：各个mall； * 2、单一项目销售额：mall的当日开始营业时间到当前时间累计销售额； */ public void projectContrast(Dataset<Row> bill, Dataset<Row> ruian,Map<String, Set<String>> ruianMallAll) { Row[] rows = (Row[])bill .filter("billType=1 and receivableAmount>0") .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter") .groupBy("name", "area_cn") .agg(functions.sum("receivableAmount").alias("mallDaySale"), functions.count("receivableAmount").alias("mallDayBillNum")) // .orderBy("area_cn") .select("name","area_cn","mallDaySale","mallDayBillNum") .collect(); Map<String,List<AreaProjSale>> rs=new HashMap<>(); if(rows.length>0) { for (Row row : rows) { String mallName=null; String areaCn=null; double mallDaySale=0D; long mallDayBillNum=0L; AreaProjSale obj=new AreaProjSale(); if(!row.isNullAt(0)){//项目、mall的名称 mallName=row.getString(0); obj.setN(mallName); } if(!row.isNullAt(1)){//区域名称 areaCn=row.getString(1); } if(!row.isNullAt(2)){//mall的日销售额 mallDaySale=row.getDouble(2); obj.setSa(mallDaySale); } if(!row.isNullAt(3)){//mall的日结账单数 mallDayBillNum=row.getLong(3); obj.setBn(mallDayBillNum); } //判断是否有该区域 if(rs.containsKey(areaCn)){ rs.get(areaCn).add(obj); }else{//不存在该区域 List<AreaProjSale> list=new ArrayList<>(); list.add(obj); rs.put(areaCn, list); } } } //填充未产生账单的数据，默认0 //获取全部瑞安的区域和mallName的集合 Set<Entry<String,Set<String>>> entrySet = ruianMallAll.entrySet(); for(Map.Entry<String, Set<String>> entry:entrySet){ String areaCn=entry.getKey();//获取区域名称 //获取每个区域的标准mall的集合 Set<String> mallSet=entry.getValue(); if(rs.containsKey(areaCn)){//实际数据中已经存在该区域相关数据 //判断实际数据mall是否完整 //用来存放实际数据中mallname的集合 Set<String> mallSetCur=new HashSet<>(); //用来存放没有账单的mall的集合 Set<String> mallSetNew=new HashSet<>(); //遍历该区域下实际数据集合 并填充set for(AreaProjSale obj:rs.get(areaCn)){ mallSetCur.add(obj.getN()); } //求两个set的差集合 将标准数据放入 mallSetNew.addAll(mallSet); //求差集合 标准数据-实际数据 得到差集 mallSetNew.removeAll(mallSetCur); //遍历差集合 ，填充默认值 mallSetNew.forEach(mn->{ rs.get(areaCn).add(new AreaProjSale(mn)); }); }else{//该区域不存在 //便利该区域下标准mall集合，逐个放入 List<AreaProjSale> list=new ArrayList<>(); mallSet.forEach(mn->{ list.add(new AreaProjSale(mn)); }); rs.put(areaCn, list); } } //各区域销售额占比 areaSaleProportion(rs); //各区域单均消费 projectAvgConsumer(rs); //转成json Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>(); pmap.put("各项目销售额对比", rs); //转成json String str=JSONObject.toJSONString(pmap); System.out.println("====####--projectContrast##"+str); //1 8各项目销售额对比dpc_8 RedisUtil.setObject("dpc_8", str); } /** * 所有mall中销售额最高的top10店铺 * @param bill * @param shop * @param ruian */ public void saleForShopTop10(Dataset<Row> bill, Dataset<Row> shop, Dataset<Row> ruian) { Row[] rows = (Row[])bill .filter("billType=1 and receivableAmount>0") .join(shop, bill.col("shopEntityId").equalTo(shop.col("shop_entity_id")), "leftouter") .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter") .groupBy("shop_entity_name", "name") .agg(functions.sum("receivableAmount").alias("shopDaySale")) .orderBy(new Column("shopDaySale").desc()) .select("shop_entity_name","name","shopDaySale") .limit(10) .head(10); List<ShopSaleRank> ssrList=new LinkedList<>(); if(rows.length>0) { for (Row row : rows) { ShopSaleRank ssr=new ShopSaleRank(); if(!row.isNullAt(0)){//店铺名称 ssr.setSn(row.getString(0)); } if(!row.isNullAt(1)){//项目/mall名称 ssr.setPn(row.getString(1)); } if(!row.isNullAt(2)){//店铺日销售额 ssr.setSa(row.getDouble(2)); } ssrList.add(ssr); } } //转成json Map<String, List<ShopSaleRank>> pmap=new HashMap<>(); pmap.put("店铺销售排行", ssrList); String str=JSON.toJSONString(pmap); System.out.println("====####--saleForShopTop10##"+str); //将10个店铺日销售额放入redis RedisUtil.setObject("dsfst_9",str); } /** * 各项目业态销售额对比 * * @param session * @param beginYMDH * @param endYMDH */ public void projectTypeSaleContrast(Dataset<Row> bill, Dataset<Row> shop, Dataset<Row> type, Dataset<Row> ruian, Set<String> areaSet,Set<String> mallSet,Set<String> ruianTypeAll) { Row[] rows = (Row[])bill .filter("billType=1 and receivableAmount>0") .join(shop, shop.col("shop_entity_id").equalTo(bill.col("shopEntityId")), "leftouter") .join(type, type.col("id").equalTo(shop.col("shop_entity_type_root")), "leftouter") .join(ruian, ruian.col("rmid").equalTo(bill.col("shopId"))) .groupBy("name", "shop_type_name") .agg(functions.sum("receivableAmount").alias("shopTypeDaySale"), functions.countDistinct("shop_entity_id").alias("shopNum")) .select("name","shop_type_name","shopTypeDaySale","shopNum") .collect(); Map<String,List<ProjectTypeShopNumber>> map=new HashMap<>(); if(rows.length>0) { for (Row row : rows) { ProjectTypeShopNumber pac=new ProjectTypeShopNumber(); String mallName=null; if(!row.isNullAt(0)){//项目、mall名称 mallName=row.getString(0); } if(!row.isNullAt(1)){//业态名称 pac.setN(row.getString(1)); }else{ pac.setN("其它"); } if(!row.isNullAt(2)){//mall的业态日销售额 pac.setSa(row.getDouble(2)); } if(!row.isNullAt(3)){//mall的业态店铺数量 pac.setSn(row.getLong(3)); } if(map.containsKey(mallName)){//更新map中的list列表 List<ProjectTypeShopNumber> pacList=map.get(mallName); pacList.add(pac); }else{//新的mall 新建列表放入map List<ProjectTypeShopNumber> pacList=new LinkedList<>(); pacList.add(pac); map.put(mallName, pacList); } } } //为没有产生账单的业态或mall填充默认数据，是数据看起来完整 mallSet.forEach(mallName->{ if(!map.containsKey(mallName)) {//没有账单的mall List<ProjectTypeShopNumber> list=new ArrayList<>(); //遍历全量业态，进行填充 ruianTypeAll.forEach(type_->{ list.add(new ProjectTypeShopNumber(type_,0.0,0)); }); map.put(mallName,list); }else{//有账单的mall //用来获取实际账单数据中已存在的业态 Set<String> ruianTypeCur = new HashSet<>(); //用来存放没有账单的业态 Set<String> ruianTypeNew = new HashSet<>(); //遍历数据，提取实际数据中的业态 for(ProjectTypeShopNumber obj:map.get(mallName)){ ruianTypeCur.add(obj.getN()); } //将产生账单的业态和全量业态求差即 ruianTypeNew.addAll(ruianTypeAll); ruianTypeNew.removeAll(ruianTypeCur); //将没有实际账单的业态数据填从（默认值填充） for(String type_:ruianTypeNew){ map.get(mallName).add(new ProjectTypeShopNumber(type_,0.0,0)); } } }); //为==各项目店铺数量==填充数据 projectShopNumber(map); //转成json Map<String, Map<String,List<ProjectTypeShopNumber>>> pmap=new HashMap<>(); pmap.put("各项目业态销售额对比", map); String str=JSON.toJSONString(pmap); System.out.println("====####--projectTypeSaleContrast##"+str); RedisUtil.setObject("dptsc_11",str); } //============================================================================================================= /** * 结账单数 * * @param tsb */ public void totalSettlementBillNumber(TotalSettlementBillNumber tsb) { Map<String, TotalSettlementBillNumber> map=new HashMap<>(); map.put("结账单数", tsb); String str=JSONObject.toJSONString(map); System.out.println("====####--totalSettlementBillNumber##"+str); //将结账单数放入redis RedisUtil.setObject("dtsbn_6", str); } /** * 各区域销售占比 * 直接将数据封装成json,无需额外处理 * * @param rs 已经封装好的数据，请参考{@link com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService.projectContrast()} */ private void areaSaleProportion(Map<String, List<AreaProjSale>> rs) { Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>(); pmap.put("各区域销售额占比 ", rs); //转成json String str=JSONObject.toJSONString(pmap); System.out.println("====####--areaSaleProportion##"+str); //各区域销售占比dasp_1 RedisUtil.setObject("dasp_1", str); } /** * 各项目单均消费 * 版本二 营业时间24小时制 * 1、各项目：各个mall； * 2、单均消费：mall的当日开始营业时间到当前时间累计销售额，应收额累计/结账单累计； * 直接将数据封装成json，无需额外处理 * * @param rs 已经封装好的数据，请参考{@link com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService.projectContrast()} */ private void projectAvgConsumer(Map<String, List<AreaProjSale>> rs) { Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>(); pmap.put("各项目单均消费", rs); //转成json String str=JSONObject.toJSONString(pmap); System.out.println("====####--projectAvgConsumer##"+str); //10各项目单均消费dpac_10 RedisUtil.setObject("dpac_10", str); } /** * 各项目店铺数量 * 数据填充自上面的方法 * * {@link projectTypeSaleContrast(SparkSession session,String beginYMDH,String endYMDH)} */ private void projectShopNumber(Map<String,List<ProjectTypeShopNumber>> map) { //转成json Map<String, Map<String,List<ProjectTypeShopNumber>>> pmap=new HashMap<>(); pmap.put("各项目店铺数量", map); String str=JSON.toJSONString(pmap); System.out.println("====####--projectShopNumber##"+str); RedisUtil.setObject("dpsn_12",str); } }
