TGSRec/process_amazon.py at master · DyGRec/TGSRec · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import json
import numpy as np
import pandas as pd
import gzip

def preprocess(data_name, meta_path):
    u_list, i_list, ts_list, label_list = [], [], [], []
    feat_l = []
    idx_list = []
    u_map = {}
    i_map = {}

    u_ind = 0
    i_ind = 0
    with gzip.open(data_name, 'r') as f:
        for line in f:
            e = eval(line)
            u = e['reviewerID']
            i = e['asin']

            if u not in u_map:
                u_map[u] = u_ind
                u_ind += 1
            if i not in i_map:
                i_map[i] = i_ind
                i_ind += 1

    i_meta_map = {}
    #with gzip.open(meta_path, 'r') as f:
    #    for line in f:
    #        e = eval(line)
    #        if e['asin'] not in i_meta_map:
    #            i_meta_map[e['asin']] = e.get('title', '')

    data = []
    with gzip.open(data_name,'r') as f:
        for line in f:
            e = eval(line)
            data.append([e['reviewerID'], e['asin'], float(e['unixReviewTime'])])
    sorted_data = sorted(data, key=lambda x:x[2])

    for idx, eachinter in enumerate(sorted_data):
        u = u_map[eachinter[0]]
        i = i_map[eachinter[1]]
        ts = eachinter[2]
        feat = np.array([0 for _ in range(8)])
        u_list.append(u)
        i_list.append(i)
        ts_list.append(ts)
        label_list.append(1)
        idx_list.append(idx)

        feat_l.append(feat)

    user_ind_id_map = {v:k for k, v in u_map.items()}
    item_ind_id_map = {v:{'item_id': k, 'title': i_meta_map.get(k, '')} for k, v in i_map.items()}
    return pd.DataFrame({'u': u_list,
                         'i':i_list,
                         'ts':ts_list,
                         'label':label_list,
                         'idx':idx_list}), np.array(feat_l), user_ind_id_map, item_ind_id_map


def reindex(df):
    assert(df.u.max() - df.u.min() + 1 == len(df.u.unique()))
    assert(df.i.max() - df.i.min() + 1 == len(df.i.unique()))

    upper_u = df.u.max() + 1
    new_i = df.i + upper_u

    new_df = df.copy()
    print(new_df.u.max())
    print(new_df.i.max())

    new_df.i = new_i
    new_df.u += 1
    new_df.i += 1
    new_df.idx += 1

    print(new_df.u.max())
    print(new_df.i.max())

    return new_df


def run(data_name):
    PATH = '/home/zfan/BDSC/projects/datasets/reviews_{}_5.json.gz'.format(data_name)
    meta_path = '/home/zfan/BDSC/projects/datasets/metadata/meta_{}.json.gz'.format(data_name)
    OUT_DF = './processed/ml_{}.csv'.format(data_name)
    OUT_FEAT = './processed/ml_{}.npy'.format(data_name)
    OUT_NODE_FEAT = './processed/ml_{}_node.npy'.format(data_name)
    u_map_file = './processed/{}_u_map.json'.format(data_name)
    i_map_file = './processed/{}_i_map.json'.format(data_name)

    df, feat, u_ind_id_map, i_ind_id_map = preprocess(PATH, meta_path)
    with open(u_map_file, 'w') as f:
        f.write(json.dumps(u_ind_id_map, sort_keys=True, indent=4))
    with open(i_map_file, 'w') as f:
        f.write(json.dumps(i_ind_id_map, sort_keys=True, indent=4))
    print("#u", len(u_ind_id_map))
    print("#i", len(i_ind_id_map))

    new_df = reindex(df)

    print(feat.shape)
    empty = np.zeros(feat.shape[1])[np.newaxis, :]
    feat = np.vstack([empty, feat])

    max_idx = max(new_df.u.max(), new_df.i.max())
    rand_feat = np.zeros((max_idx + 1, feat.shape[1]))

    print(feat.shape)
    new_df.to_csv(OUT_DF, index=False)
    np.save(OUT_FEAT, feat)
    np.save(OUT_NODE_FEAT, rand_feat)


#run('Tools_and_Home_Improvement')
#run('Office_Products')
#run('Toys_and_Games')
#run('Digital_Music')
#run('Baby')