以下文章使用了kafka作为storm的sport数据源,依赖于docker-compose环境,如果自己本机已经有了
zookeeper与kafka环境则可以使用自己的环境。
使用教程
docker-compose.yml
version: '2'services: zookeeper: image: wurstmeister/zookeeper ports: - "2181:2181" kafka: image: wurstmeister/kafka ports: - "9092" environment: KAFKA_ADVERTISED_HOST_NAME: 192.168.1.186 KAFKA_ZOOKEEPER_CONNECT: 192.168.1.186:2181 volumes: - /var/run/docker.sock:/var/run/docker.sock
build.gradle
dependencies {
compile group: 'org.apache.storm', name: 'storm-core', version: '1.2.2'
compile group: 'org.apache.storm', name: 'storm-kafka-client', version: '1.2.2'}Application.java
public class Application {
public static void main(String[] args) throws Exception {
KafkaSpoutConfig.Builder<String,String> kafkaBuild = KafkaSpoutConfig.builder("192.168.1.186:32770","test","test1","test2");
kafkaBuild.setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.UNCOMMITTED_LATEST);
kafkaBuild.setOffsetCommitPeriodMs(100);//设置多长时间向kafka提交一次offset
kafkaBuild.setProp(ConsumerConfig.GROUP_ID_CONFIG,"testGroup");
kafkaBuild.setProp(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG,1);
kafkaBuild.setProp(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG,0);
KafkaSpoutConfig<String,String> build = kafkaBuild.build();
KafkaSpout<String,String> kafkaSpout = new KafkaSpout<>(build);
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("kafkaSport",kafkaSpout,1);
builder.setBolt("print-bolt",new PrintBolt(),1)
.shuffleGrouping("kafkaSport");
Config config = new Config();
config.setNumWorkers(1); if(args.length==0){
config.setDebug(true);
LocalCluster localCluster = new LocalCluster();
localCluster.submitTopology("test",config,builder.createTopology());
}else{
config.setDebug(false);
StormSubmitter.submitTopology("test",config,builder.createTopology());
}
}
}PrintBolt.java
public class PrintBolt extends BaseRichBolt { private OutputCollector collector; @Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {} @Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector;
} @Override
public void execute(Tuple input) {
String topic = input.getString(0); long messageOffet = input.getLong(2);
String content = input.getString(4);
System.out.println(content);
collector.ack(input);
}
}往kafka发送一个消息
docker exec -ti 4f27fbb6655c bash /opt/kafka/bin/kafka-console-producer.sh --broker-list 192.168.1.186:32770 --topic test
输入消息回车即可看到程序输出
root@localhost /h/lake# docker exec -ti 4f27fbb6655c bash /opt/kafka/bin/kafka-console-producer.sh --broker-list 192.168.1.186:32770 --topic test>hello world. >
输出
...312034 [Thread-20-print-bolt-executor[3 3]] INFO o.a.s.d.executor - Processing received message FOR 3 TUPLE: source: kafkaSport:2, stream: default, id: {-3468855975737070311=-8649143164499739979}, [test, 0, 16, null, hello world.]312034 [Thread-22-__acker-executor[1 1]] INFO o.a.s.d.executor - Processing received message FOR 1 TUPLE: source: kafkaSport:2, stream: __ack_init, id: {}, [-3468855975737070311 -8649143164499739979 2]
hello world.312034 [Thread-20-print-bolt-executor[3 3]] INFO o.a.s.d.task - Emitting: print-bolt __ack_ack [-3468855975737070311 -8649143164499739979]312034 [Thread-22-__acker-executor[1 1]] INFO o.a.s.d.executor - BOLT ack TASK: 1 TIME: -1 TUPLE: source: kafkaSport:2, stream: __ack_init, id: {}, [-3468855975737070311 -8649143164499739979 2]
...
作者:dounine
链接:https://www.jianshu.com/p/408b3a4fc92c
點擊查看更多內容
為 TA 點贊
評論
評論
共同學習,寫下你的評論
評論加載中...
作者其他優質文章
正在加載中
感謝您的支持,我會繼續努力的~
掃碼打賞,你說多少就多少
贊賞金額會直接到老師賬戶
支付方式
打開微信掃一掃,即可進行掃碼打賞哦