Python-code-sample/scraping_ga_api_url.py at master · simonbreton/Python-code-sample · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
from oauth2client.service_account import ServiceAccountCredentials
from apiclient.discovery import build
from oauth2client import client
from oauth2client import tools
from oauth2client import file
from bs4 import BeautifulSoup
from urlparse import urljoin
import numpy as np
import httplib2
import argparse
import urllib2
import csv


def get_service(api_name, api_version, scope, key_file_location,
                service_account_email):
  """Get a service that communicates to a Google API.
  Args:
    api_name: The name of the api to connect to.
    api_version: The api version to connect to.
    scope: A list auth scopes to authorize for the application.
    key_file_location: The path to a valid service account p12 key file.
    service_account_email: The service account email address.
  Returns:
    A service that is connected to the specified API.
  """

  credentials = ServiceAccountCredentials.from_p12_keyfile(
    service_account_email, key_file_location, scopes=scope)

  http = credentials.authorize(httplib2.Http())

  # Build the service object.
  service = build(api_name, api_version, http=http)

  return service

def get_first_profile_id(service):
  # Declaring variable
  profiles_list = []
  ua_list = []
  url_list = []
  column_url = []
  column_share_k = []
  column_share = []

  # Hostname to complete page url in order to scrap them properly
  base = 'http://www.hostname.com'

  # Accessing property
  properties = service.management().webproperties().list(
    accountId='~all').execute()

  # Building property list
  for property in properties.get('items', []):
    ua_list.append(property.get('id'))

  #looping through ua list
  for ua in ua_list:
    profiles = service.management().profiles().list(
        accountId='xxxxxxxxx',
        webPropertyId= ua ).execute()

  # Building a list of Profile id
    for profile in profiles.get('items', []):
      profiles_list.append(profile.get('id'))

  # Loop through the profiles_list and get the best pages for each profile
  for profile in profiles_list:
    response = service.data().ga().get(
      ids='ga:' + profile,
      start_date='1daysAgo',
      end_date='today',
      metrics='ga:sessions',
      dimensions='ga:pagePath',
      sort='-ga:sessions',
      filters='ga:sessions>400').execute()

    url_list.extend(row[0] for row in response.get('rows', []))

    # Building a list of full url (Hostname + Page path)
    fullurl = [urljoin(base, h) for h in url_list]

    # Scraping some data from the url list
    for url in fullurl:

      try:
          page = urllib2.urlopen(url)
      except urllib2.HTTPError as e:
              if e.getcode() == 404: # eheck the return code
                  print url
                  continue
      soup = BeautifulSoup(page, 'html.parser')

      # Take out the <div> of name and get its value
      name_box = soup.find(attrs={'class': 'nb-shares'})
      if name_box is None:
        continue
      share_count = name_box.text.strip() # strip() is used to remove starting and trailing

      # save the data in tuple
      column_url.append(url)
      column_share_k.append(share_count)

      # Format the data scraped
      column_share = [int(1000*float(x.replace('k', ''))) if 'k' in x else int(x) for x in column_share_k]

    #export in csv
    csv_out = open(response.get('profileInfo').get('profileName') + '.csv', 'wb')
    mywriter = csv.writer(csv_out)
    for row in zip(column_url, column_share):
      mywriter.writerow([row])
    csv_out.close()

    #reset list
    fullurl = []
    url_list = []
    share_count = []
    column_share_k = []
    column_url = []
    column_share = []

def main():
  # Define the auth scopes to request.
  scope = ['https://www.googleapis.com/auth/analytics.readonly']

  # Use the developer console and replace the values with your
  # service account email and relative location of your key file.
  service_account_email = 'email@nameproject.iam.gserviceaccount.com'
  key_file_location = 'yourfilename.p12'

  # Authenticate and construct service.
  service = get_service('analytics', 'v3', scope, key_file_location,
    service_account_email)
  profile = get_first_profile_id(service)

if __name__ == '__main__':
  main()