LogServiceでOSSへデータを集める方法

pip3.6 install -U aliyun-log-python-sdk
pip3.6 install twitter

# -*- coding: utf-8 -*-

from aliyun.log.logitem import LogItem
from aliyun.log.logclient import LogClient
from aliyun.log.putlogsrequest import PutLogsRequest

import time
import twitter

SCREEN_NAME = 'write here'

# OAuth こちらはTwitter Developerに申請する必要があります。
# https://developer.twitter.com/content/developer-twitter/ja.html
ACCESS_TOKEN_KEY = ''
ACCESS_TOKEN_SECRET = ''
CONSUMER_KEY = ''
CONSUMER_SECRET = ''

endpoint = 'ap-northeast-1.log.aliyuncs.com'
accessKeyId = '<自分のaccessKeyId>'
accessKey = '<自分のaccessKey>'
project = 'bigdata-prod-tech'
logstore = 'twitter-logstore'
token = ""

def convert_to_text(row_value):
    if type(row_value) is None:
        return ""

    elif type(row_value) is str:
        return row_value

    elif type(row_value) is int:
        return str(row_value)

    elif type(row_value) is float:
        return str(row_value)    

    elif type(row_value) is bool:
        if row_value is True:
            return "True"   
        else:
            return "False"        
    else:
        return ""


def twitter_put_logs():

    client = LogClient(endpoint, accessKeyId, accessKey, token)     
    oauth = twitter.OAuth(ACCESS_TOKEN_KEY,
                      ACCESS_TOKEN_SECRET,
                      CONSUMER_KEY,
                      CONSUMER_SECRET)

    twitter_api = twitter.Twitter(auth=oauth)
    friends = twitter_api.friends.ids(screen_name=SCREEN_NAME, count=5000)
    friends_ids = ','.join(map(str, friends['ids']))

    stream = twitter.TwitterStream(auth=oauth, secure=True)
    for tweet in stream.statuses.filter(follow=friends_ids):
        created_at = convert_to_text(tweet['created_at'])
        _id = convert_to_text(tweet['id'])
        id_str = convert_to_text(tweet['id_str'])
        text = convert_to_text(tweet['text'])
        source = convert_to_text(tweet['source'])
        in_reply_to_status_id = convert_to_text(tweet['in_reply_to_status_id'])
        in_reply_to_status_id_str = convert_to_text(tweet['in_reply_to_status_id_str'])
        in_reply_to_user_id = convert_to_text(tweet['in_reply_to_user_id'])
        in_reply_to_user_id_str = convert_to_text(tweet['in_reply_to_user_id_str'])
        in_reply_to_screen_name = convert_to_text(tweet['in_reply_to_screen_name'])
        user_id = convert_to_text(tweet['user']['id'])
        user_id_str = convert_to_text(tweet['user']['id_str'])
        user_name = convert_to_text(tweet['user']['name'])
        user_screen_name = convert_to_text(tweet['user']['screen_name'])
        user_location = convert_to_text(tweet['user']['location'])
        user_url = convert_to_text(tweet['user']['url'])
        user_description = convert_to_text(tweet['user']['description'])
        user_followers_count = convert_to_text(tweet['user']['followers_count'])
        user_friends_count = convert_to_text(tweet['user']['friends_count'])
        user_listed_count = convert_to_text(tweet['user']['listed_count'])
        user_favourites_count = convert_to_text(tweet['user']['favourites_count'])
        user_statuses_count = convert_to_text(tweet['user']['statuses_count'])
        user_created_at = convert_to_text(tweet['user']['created_at'])

        contents = [
            ( 'created_at', created_at),
            ( 'id', _id),
            ( 'text', text),                
            ( 'source', source),
            ( 'in_reply_to_status_id', in_reply_to_status_id),
            ( 'in_reply_to_user_id', in_reply_to_user_id),
            ( 'in_reply_to_screen_name', in_reply_to_screen_name),
            ( 'user_id', user_id),
            ( 'user_name', user_name),
            ( 'user_screen_name', user_screen_name),
            ( 'user_location', user_location),
            ( 'user_url', user_url),                        
            ( 'user_description', user_description),
            ( 'user_followers_count', user_followers_count),
            ( 'user_friends_count', user_friends_count),
            ( 'user_listed_count', user_listed_count),   
            ( 'user_favourites_count', user_favourites_count),
            ( 'user_statuses_count', user_statuses_count),
            ( 'user_created_at', user_created_at)
         ]

        topic = 'twitter_log'
        source = 'source'

        logitemList = [] 
        logItem = LogItem()
        logItem.set_time(int(time.time()))
        logItem.set_contents(contents)
        logitemList.append(logItem)
        request = PutLogsRequest(project, logstore, topic, source, logitemList)
     
        response = client.put_logs(request)
        response.log_print()


if __name__=='__main__':
    twitter_put_logs()