Experimenting with Kafka Streams using Twitter API to filter out follower count
Create topic
❯ kafka-topics --zookeeper localhost:2181 --topic popular_tweets --create --partitions 3 --replication-factor 1
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
Created topic popular_tweets.Run the producer from this repo then run the stream
[experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1] INFO org.apache.kafka.clients.consumer.internals.SubscriptionState - [Consumer clientId=experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1-consumer, groupId=experimenting-with-kafka-streams] Resetting offset for partition tweets-0 to position FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[localhost:9092 (id: 0 rack: null)], epoch=0}}.
[experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1] INFO org.apache.kafka.clients.consumer.internals.SubscriptionState - [Consumer clientId=experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1-consumer, groupId=experimenting-with-kafka-streams] Resetting offset for partition tweets-3 to position FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[localhost:9092 (id: 0 rack: null)], epoch=0}}.
[experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1] INFO org.apache.kafka.clients.consumer.internals.SubscriptionState - [Consumer clientId=experimenting-with-kafka-streams-066f70dd-505f-478b-9d04-973a9c036b89-StreamThread-1-consumer, groupId=experimenting-with-kafka-streams] Resetting offset for partition tweets-2 to position FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[localhost:9092 (id: 0 rack: null)], epoch=0}}.Run the consumer with the topic popular_tweets
❯ kafka-console-consumer --bootstrap-server localhost:9092 --topic popular_tweets
{"created_at":"Thu May 20 00:22:38 +0000 2021","id":1395173083045105673,"id_str":"1395173083045105673","text":"RT @WSBGiveaways: $500 Giveaway to 5 Random People!\n\nRetweet,Like, Follow @JustInCrypt0 & Myself!\n\nEnds in 5 days!\n\n#WSBGIVEAWAYS #cryptocu\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1389626650300661760,"id_str":"1389626650300661760","name":"Wall Street Bets Giveaways #WSBWOLFS\ud83d\udc3a","screen_name":"WSBGiveaways","location":"$1,620 Given $998,380 To Go\ud83d\ude80","url":"https:\/\/www.instagram.com\/wsbgiveaways\/","description":"#WSBWOLFS\ud83d\udc3a #Cash, #Stocks, #BNB #Crypto #NFT #Giveaways |@elonmusk Made His Own Stimulus For Us #Doge \ud83d\udc15 |Vouch:#WSBGIVEAWAYS","translator_type":"none","protected":false,"verified":false,"followers_count":12049,"friends_count":1,"listed_count":6,"favourites_count":312,"statuses_count":439,"created_at":"Tue May 04 17:03:16 +0000 2021","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1389701705747607556\/VQ5Yg3TX_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1389701705747607556\/VQ5Yg3TX_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1389626650300661760\/1620251156","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Mon May 17 00:37:14 +0000 2021","id":1394089591720796163,"id_str":"1394089591720796163","text":"$500 Giveaway to 5 Random People!\n\nRetweet,Like, Follow @JustInCrypt0 & Myself!\n\nEnds in 5 days!\n\n#WSBGIVEAWAYS\u2026 https:\/\/t.co\/34Z43x3ec1","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1389626650300661760,"id_str":"1389626650300661760","name":"Wall Street Bets Giveaways #WSBWOLFS\ud83d\udc3a","screen_name":"WSBGiveaways","location":"$1,620 Given $998,380 To Go\ud83d\ude80","url":"https:\/\/www.instagram.com\/wsbgiveaways\/","description":"#WSBWOLFS\ud83d\udc3a #Cash, #Stocks, #BNB #Crypto #NFT #Giveaways |@elonmusk Made His Own Stimulus For Us #Doge \ud83d\udc15 |Vouch:#WSBGIVEAWAYS","translator_type":"none","protected":false,"verified":false,"followers_count":12049,"friends_count":1,"listed_count":6,"favourites_count":312,"statuses_count":438,"created_at":"Tue May 04 17:03:16 +0000 2021","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1389701705747607556\/VQ5Yg3TX_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1389701705747607556\/VQ5Yg3TX_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1389626650300661760\/1620251156","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"$500 Giveaway to 5 Random People!\n\nRetweet,Like, Follow @JustInCrypt0 & Myself!\n\nEnds in 5 days!\n\n#WSBGIVEAWAYS #cryptocurrency #bnb #Bitcoin #dogecoin #GiveawayTime #wsb \n\n\ud83d\ude80\ud83d\udcb8","display_text_range":[0,180],"entities":{"hashtags":[{"text":"WSBGIVEAWAYS","indices":[102,115]},{"text":"cryptocurrency","indices":[116,131]},{"text":"bnb","indices":[132,136]},{"text":"Bitcoin","indices":[137,145]},{"text":"dogecoin","indices":[147,156]},{"text":"GiveawayTime","indices":[157,170]},{"text":"wsb","indices":[171,175]}],"urls":[],"user_mentions":[{"screen_name":"JustInCrypt0","name":"JustInCrypto","id":1679768760,"id_str":"1679768760","indices":[56,69]}],"symbols":[]}},"quote_count":10,"reply_count":1042,"retweet_count":2896,"favorite_count":2626,"entities":{"hashtags":[{"text":"WSBGIVEAWAYS","indices":[102,115]}],"urls":[{"url":"https:\/\/t.co\/34Z43x3ec1","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1394089591720796163","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[{"screen_name":"JustInCrypt0","name":"JustInCrypto","id":1679768760,"id_str":"1679768760","indices":[56,69]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"WSBGIVEAWAYS","indices":[120,133]}],"urls":[],"user_mentions":[{"screen_name":"WSBGiveaways","name":"Wall Street Bets Giveaways #WSBWOLFS\ud83d\udc3a","id":1389626650300661760,"id_str":"1389626650300661760","indices":[3,16]},{"screen_name":"JustInCrypt0","name":"JustInCrypto","id":1679768760,"id_str":"1679768760","indices":[74,87]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en","timestamp_ms":"1621470158717"}
# ...Check the filter stream works by confirming the followers_count
{
"created_at": "Thu May 20 00:22:38 +0000 2021",
"user": {
"followers_count": 12049
}
}