try pull request

.gitignore

@@ -10,7 +10,6 @@ dist
 build
 eggs
 parts
-bin
 var
 sdist
 develop-eggs

README.md

@@ -1,4 +1,10 @@
-#Guide
+# Why
+
+[Pythonic.info](http://pythonic.info) is a [HackerNews](http://news.ycombinator.com) clone with python/tornado
+
+It's also a demo of [NoMagic](https://github.com/kernel1983/NoMagic) data framework, showing that [NoMagic](https://github.com/kernel1983/NoMagic) is actually working in production enviorment.
+
+# Guide
 
 ## How to
 ### requirements

bin/delete.py

@@ -0,0 +1,55 @@
+import sys
+import datetime
+import time
+import pickle
+import uuid
+import binascii
+import json
+import zlib
+
+sys.path.append(".")
+sys.path.append("..")
+
+from setting import settings
+from setting import conn
+#from setting import ring
+
+import nomagic
+import nomagic.feeds
+
+
+def get_comments(comments):
+    comment_ids = []
+    for comment in comments:
+        assert comment.get("type") == "comment"
+        #comment["like_count"] = len(comment.get("likes", []))
+        #comment["like"] = self.user_id in set(comment.get("likes", [])) if self.current_user else False
+        #comment["comment_count"] = 0
+        #print comment["comments"] if comment.get("comments") else []
+        comment_ids.append(comment["id"])
+        comment_ids.extend(get_comments(comment["comments"]) if comment.get("comments") else [])
+
+    return comment_ids
+
+if len(sys.argv) < 2:
+    sys.exit()
+
+print sys.argv[1]
+activity_id = sys.argv[1]
+
+activity = nomagic._get_entity_by_id(activity_id)
+print activity
+assert activity.get("type") == "status"
+
+comments, user_ids = nomagic.feeds.get_comments(activity)
+comment_ids = get_comments(comments)
+print comment_ids
+
+# delete comment_ids
+for comment_id in comment_ids:
+    nomagic._node(comment_id).execute_rowcount("DELETE FROM entities WHERE id = %s", comment_id)
+
+# delete activity_id
+nomagic._node(activity_id).execute_rowcount("DELETE FROM entities WHERE id = %s", activity_id)
+
+conn.execute_rowcount("DELETE FROM index_posts WHERE entity_id = %s", activity_id)

bin/email_daily.py

@@ -0,0 +1,81 @@
+import sys
+import os
+import datetime
+import time
+import pickle
+import uuid
+import binascii
+import json
+import zlib
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../')
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../vendor')
+
+import tornado.template
+import tornado.locale
+import amazon_ses
+
+from setting import settings
+from setting import conn
+
+import nomagic
+
+
+def daily(hours):
+    now = time.time()
+    offset = 0
+    post_ids_to_email = set()
+    while True:
+        index_posts = conn.query("SELECT * FROM index_posts ORDER BY rank DESC LIMIT %s, 100", offset)
+
+        if len(index_posts) == 0:
+            break
+
+        post_ids = [post["entity_id"] for post in index_posts]
+        for post_id, post in nomagic._get_entities_by_ids(post_ids):
+            period = (now - time.mktime(datetime.datetime.strptime(post["datetime"], "%Y-%m-%dT%H:%M:%S.%f").timetuple())) / 3600
+
+            if period <= hours:
+                post_ids_to_email.add(post_id)
+
+        offset += 100
+
+    posts_to_email = nomagic._get_entities_by_ids(post_ids_to_email)
+    loader = tornado.template.Loader(os.path.dirname(os.path.abspath(__file__)) + "/../template/")
+
+    locale = tornado.locale.get()
+    msg = amazon_ses.EmailMessage()
+    msg.subject = locale.translate('Pythonic Info Daily').encode("utf-8")
+    msg.bodyHtml = loader.load("email_daily.html").generate(posts=posts_to_email, _=locale.translate)
+
+    users = []
+    users_not_to_send = []
+    users_exists = conn.query("SELECT * FROM index_login")
+    for user_id, user in nomagic._get_entities_by_ids([user_exists["entity_id"] for user_exists in users_exists]):
+        if user.get("receive_daily_email", True):
+            users.append(user)
+        else:
+            users_not_to_send.append(user)
+
+    users_invited = conn.query("SELECT * FROM invite")
+
+    sender = amazon_ses.AmazonSES(settings["AmazonAccessKeyID"], settings["AmazonSecretAccessKey"])
+    emails = set([user["login"] for user in users_exists] + [user["email"] for user in users_invited]) - set([user["email"] for user in users_not_to_send])
+    for email in emails:
+        if "@" in email:
+            print email
+            sender.sendEmail(settings["email_sender"], email, msg)
+
+
+if __name__ == '__main__':
+    tornado.locale.load_translations(os.path.join(os.path.dirname(__file__) + "/../csv_translations/"))
+    tornado.locale.set_default_locale("zh_CN")
+
+    if len(sys.argv) < 2:
+        sys.exit()
+
+    hours = float(sys.argv[1])
+    #print hours
+
+    daily(hours)
+

bin/migration.py

@@ -0,0 +1,44 @@
+import sys
+import pickle
+import uuid
+import binascii
+import json
+import zlib
+
+sys.path.append(".")
+sys.path.append("..")
+
+from setting import settings
+from setting import conn
+from setting import ring
+
+import nomagic
+from nomagic import _RING
+
+_NUMBER = len(ring)
+
+def _number(key): return int(key, 16) % _NUMBER
+
+def _node(key): return ring[_RING.get_node(key)]
+
+
+for r in ring:
+    offset = 0
+    ids_to_delete = []
+    while True:
+        users = r.query("SELECT * FROM entities ORDER BY auto_increment LIMIT %s, 100", offset)
+
+        if len(users) == 0:
+            break
+
+        for user in users:
+            user_id = str(user["id"])
+            print user["id"], _number(user_id), nomagic._RING.get_node(user_id)
+            if r is not _node(user_id):
+                _node(user_id).execute_rowcount("INSERT INTO entities (id, body) VALUES (%s, %s)", user["id"], user["body"])
+                ids_to_delete.append(user_id)
+
+        offset += 100
+
+    for id_to_delete in ids_to_delete:
+        print r.execute_rowcount("DELETE FROM entities WHERE id = %s", id_to_delete)

bin/rank.py

@@ -0,0 +1,45 @@
+import sys
+import os
+import datetime
+import time
+import pickle
+import uuid
+import binascii
+import json
+import zlib
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../')
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../vendor')
+
+from setting import settings
+from setting import conn
+
+import nomagic
+
+
+def rank(points, period): return int( (points + 1) /  ( (period + 2) ** 1.8) * 1000000000 )
+
+def ranking():
+    now = time.time()
+    offset = 0
+    while True:
+        index_posts = conn.query("SELECT * FROM index_posts ORDER BY rank DESC LIMIT %s, 100", offset)
+
+        if len(index_posts) == 0:
+            break
+
+        post_ids = [post["entity_id"] for post in index_posts]
+        for post_id, post in nomagic._get_entities_by_ids(post_ids):
+            period = (now - time.mktime(datetime.datetime.strptime(post["datetime"], "%Y-%m-%dT%H:%M:%S.%f").timetuple())) / 3600
+            points = len(post["likes"])
+            post_rank = rank(points, period)
+
+            conn.execute("UPDATE index_posts SET rank = %s WHERE entity_id = %s", post_rank, post_id)
+
+        offset += 100
+
+if __name__ == '__main__':
+    #print rank(0, 0)
+    #print rank(3000, 0)
+    ranking()
+