MariaDB Migration

VERSION

@@ -1 +1 @@
-20
+21

front_end/Containerfile

@@ -74,5 +74,7 @@ RUN bash /app/build_html.sh
 ADD front_end/handlers/*.py /app/
 ADD front_end/*.py /app/
 
+RUN bash /app/build_html.sh
+
 #ENTRYPOINT ["bash"]
 ENTRYPOINT ["bash", "/app/startup.sh"]

front_end/handlers/BaseUserHandler.py

@@ -6,7 +6,7 @@
 class BaseUserHandler(RequestHandler):
     def prepare(self):
         self.settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-        self.content = Content(self.settings_dict)
+        self.content = ContentSQLite(self.settings_dict)
         self.user_info_var = contextvars.ContextVar("user_info")
         self.user_is_administrator_var = contextvars.ContextVar("user_is_administrator")
         self.user_instructor_courses_var = contextvars.ContextVar("user_instructor_courses")

front_end/handlers/CASLoginHandler.py

@@ -8,7 +8,7 @@
 class CASLoginHandler(RequestHandler):
     def prepare(self):
         self.settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-        self.content = Content(self.settings_dict)
+        self.content = ContentSQLite(self.settings_dict)
 
     async def get(self):
         try:

front_end/handlers/DevelopmentLoginHandler.py

@@ -4,7 +4,7 @@
 class DevelopmentLoginHandler(RequestHandler):
     def prepare(self):
         self.settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-        self.content = Content(self.settings_dict)
+        self.content = ContentSQLite(self.settings_dict)
 
     def get(self):
         self.render("devlogin.html", courses=self.content.get_courses(False))

front_end/handlers/GoogleLoginHandler.py

@@ -5,7 +5,7 @@
 class GoogleLoginHandler(RequestHandler, GoogleOAuth2Mixin):
     def prepare(self):
         self.settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-        self.content = Content(self.settings_dict)
+        self.content = ContentSQLite(self.settings_dict)
 
     async def get(self):
         try:

front_end/handlers/HomeHandler.py

@@ -8,7 +8,7 @@ def prepare(self):
         self.user_info_var = contextvars.ContextVar("user_info")
         self.user_is_administrator_var = contextvars.ContextVar("user_is_administrator")
         self.settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-        self.content = Content(self.settings_dict)
+        self.content = ContentSQLite(self.settings_dict)
 
         try:
             user_id = self.get_secure_cookie("user_id")

front_end/html/exercise.html

@@ -307,27 +307,26 @@ <h6 style="display:inline">Using Pair Programming:</h6>
         </div>
 
         <!--Below is the code for showing the exercise title, instructions, data url and contents, test code, and hint-->
+        <div>
+            <h4>Assignment: {{ exercise_basics["assignment"]["title"] }}</h4>
+            <div class="row-container">
+                <div class="e-title-container">
+                    <h5 style="display: inline-block;">Exercise: {{ exercise_basics["title"] }}</h5>
+                </div>
 
-        <div class="row-container">
-            <div class="e-title-container">
-                <h4 style="display: inline-block;">Assignment: {{ exercise_basics["assignment"]["title"] }}</h4>
-            </div>
-            <div class="e-title-container">
-                <h5 style="display: inline-block;">Exercise: {{ exercise_basics["title"] }}</h5>
-            </div>
-
-            <div class="buttons is-pulled-right">
-            {% if is_administrator or is_instructor or is_assistant %}
-                <a class="button is-white" href="/edit_exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ exercise_basics['id'] }}">Edit exercise</a>
-            {% end %}
-            {% if prev_exercise %}
-                <a class="button is-white" href="/exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ prev_exercise['id'] }}">Previous exercise</a>
-            {% end %}
-            {% if next_exercise %}
-                <a class="button is-white" href="/exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ next_exercise['id'] }}">Next exercise</a>
-            {% end %}
+                <div class="buttons is-pulled-right">
+                {% if is_administrator or is_instructor or is_assistant %}
+                    <a class="button is-white" href="/edit_exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ exercise_basics['id'] }}">Edit exercise</a>
+                {% end %}
+                {% if prev_exercise %}
+                    <a class="button is-white" href="/exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ prev_exercise['id'] }}">Previous exercise</a>
+                {% end %}
+                {% if next_exercise %}
+                    <a class="button is-white" href="/exercise/{{ course_basics['id'] }}/{{ assignment_basics['id'] }}/{{ next_exercise['id'] }}">Next exercise</a>
+                {% end %}
+                </div>
+                <!--<button class="button is-light is-pulled-right" onclick="showLinkModal()">Share</button>-->
             </div>
-            <!--<button class="button is-light is-pulled-right" onclick="showLinkModal()">Share</button>-->
         </div>
 
         <!--Notification messages at the top of the page for when the due date has passed-->

front_end/migration_scripts/15_to_16.py

@@ -8,7 +8,7 @@
 from content import *
 
 settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-content = Content(settings_dict)
+content = ContentSQLite(settings_dict)
 
 version = read_file("/VERSION").rstrip()
 

front_end/migration_scripts/15_to_16.sql

@@ -39,19 +39,19 @@ INSERT INTO tests (course_id, assignment_id, exercise_id, code, test_instruction
               AND test_code != ""
               AND show_test_code = 0;
 
-DROP TABLE course_registration;
+DROP TABLE IF EXISTS course_registration;
 
 DELETE FROM course_registrations WHERE user_id NOT IN (SELECT user_id FROM permissions WHERE role = 'administrator' or role = 'instructor');
 
 DELETE FROM submissions WHERE user_id NOT IN (SELECT user_id FROM permissions WHERE role = 'administrator' or role = 'instructor');
 
 DELETE FROM metadata WHERE version = 5;
 
-DROP TABLE problems;
+DROP TABLE IF EXISTS problems;
 
 DELETE FROM scores WHERE user_id NOT IN (SELECT user_id FROM permissions WHERE role = 'administrator' or role = 'instructor');
 
-DROP TABLE user_assignment_start;
+DROP TABLE IF EXISTS user_assignment_start;
 
 DELETE FROM user_assignment_starts WHERE user_id NOT IN (SELECT user_id FROM permissions WHERE role = 'administrator' or role = 'instructor');
 

front_end/migration_scripts/19_to_20_migrate.sql

@@ -12,8 +12,8 @@ DELETE FROM users;
 
 -- Delete old tables
 
-DROP TABLE course_registration;
-DROP TABLE user_assignment_start;
+DROP TABLE IF EXISTS course_registration;
+DROP TABLE IF EXISTS user_assignment_start;
 
 -- Add email_address field to users table.
 

front_end/migration_scripts/20_to_21.py

@@ -0,0 +1,123 @@
+import atexit
+import sqlite3
+import sys
+import traceback
+
+sys.path.append('/app')
+from helper import *
+from content import *
+from content_maria import *
+
+settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
+content = ContentSQLite(settings_dict)
+
+version = read_file("/VERSION").rstrip()
+
+# This tells us whether the migration has already happened.
+check_sql = '''SELECT COUNT(*) AS count
+               FROM pragma_table_info("tests")
+               WHERE name = "go"'''
+
+if content.fetchone(check_sql)["count"] > 0:
+    print("***NotNeeded***")
+else:
+    with open("/migration_scripts/20_to_21.sql") as sql_file:
+        sql_statements = sql_file.read().split(";")
+
+    try:
+        with open("/logs/progress.log", "w") as progress_file:
+            # Execute migration script's sql commands
+            for sql in sql_statements:
+                progress_file.write(sql + "\n")
+                content.execute(sql)
+
+            progress_file.write('commence migration \n----------------------------------------------------------\n\n')
+
+            # Dumps the current SQLite database into a sql file that is converted into MYSQL.
+            sqlite_dump_name = content.dump_database()
+
+            # Switch to content_mariadb.
+            content = Content(settings_dict)
+
+            with open(sqlite_dump_name) as db_dump:
+                # These tables shouldn't be present in the current version, but if they slipped through, do not migrate them over to MariaDB.
+                old_tables = ['course_registration', 'user_assignment_start', 'problems']
+
+                create_statements = []
+                other_statements = []
+
+                missing_parantheses = None
+
+                # Splits sql commands on semicolons that aren't inside strings
+                for sql_command in re.split(r"(?!\B[`'][^`']*);(?![^`']*[`']\B)", db_dump.read()):
+                    sql_command = sql_command.strip()
+
+                    # Joins sql fragments together if needed.
+                    if missing_parantheses is not None:
+                        sql_command = f"{missing_parantheses};{sql_command}"
+
+                    # If sql command is missing a closing paranthesis, flag it as a fragment and start scanning for the end.
+                    if not sql_command.endswith(")"):
+                        missing_parantheses = sql_command
+                    else:
+                        missing_parantheses = None
+                        if "TABLE" in sql_command:
+                            # Adds all create statements to a list
+                            if 'CREATE TABLE' in sql_command:
+                                # Excludes any table found in 'old_tables'
+                                if all([f'`{table}`' not in sql_command for table in old_tables]):
+                                    create_statements.append(sql_command)
+                        else:
+                            # Adds all other statements to a separate list
+                            other_statements.append(sql_command)
+
+                # Specifies order that create statements should be run in
+                order = ['courses','users','assignments','exercises','submissions','tests','presubmissions','submission_outputs','help_requests','metadata','scores','course_registrations','permissions','user_assignment_starts']
+                order =  [f"CREATE TABLE IF NOT EXISTS `{o}`" for o in order]
+                ordered_create_statements = []
+
+                # Provides error logging if the order list and create statements list are out of sync (I can't picture that this would happen anymore, but it's an important bug to check for)
+                if len(order) != len(create_statements):
+                    create_statements = [x.split(" (")[0] for x in create_statements]
+
+                    greater = order if len(order) > len(create_statements) else create_statements
+                    lesser = order if len(order) < len(create_statements) else create_statements
+                    missing = list(set(greater) - set(lesser))
+
+                    # Writes the missing statements in progress.log
+                    for m in missing:
+                        progress_file.write(f"missing: {m}" + "\n")
+
+                    sys.exit(1)
+
+                # Surely there's a cleaner, more efficient way to order the create statements but this is what I used
+                for i in range(len(order)):
+                    for j in range(len(create_statements)):
+                        if create_statements[j].strip().startswith(order[i]):
+                            ordered_create_statements.append(create_statements[j])
+                            break
+
+                # Temporarily allows adding of rows before the addition of rows they might reference as foreign keys
+                content.execute("SET FOREIGN_KEY_CHECKS=0")
+
+                for c in ordered_create_statements:
+                    # Executes each create statement in order
+                    progress_file.write(c + "\n")
+                    content.execute(c)
+
+                for o in other_statements:
+                    # Executes all other statements
+                    progress_file.write(o + "\n")
+                    content.execute(o)
+
+                # Updates metadata values and returns foreign_key_checks to on
+                content.execute("INSERT INTO `metadata` VALUES(21);")
+                content.execute("SET FOREIGN_KEY_CHECKS=1")
+
+        print("***Success***")
+
+    except:
+        print(traceback.format_exc())
+
+        with open("/logs/progress.log", "a") as progress_file:
+            progress_file.write(traceback.format_exc())

front_end/migration_scripts/20_to_21.sql

@@ -0,0 +1,39 @@
+-- I included this line because in some of the migration runs I was seeing errors that 'show_test_code' (an obselete column, would be useful to track down each instance and remove it from the code entirely given the time)
+-- was having non integers assigned to it. I didn't have the time to fully track down those errors or why an empty string would be assigned to show_test_code in the first place, so I'm leaving this here as is.
+
+-- All this is doing is changing the type of show_test_code to text
+CREATE TABLE IF NOT EXISTS exercises2 (
+                        course_id integer NOT NULL,
+                        assignment_id integer NOT NULL,
+                        exercise_id integer PRIMARY KEY AUTOINCREMENT,
+                        title text NOT NULL,
+                        visible integer NOT NULL,
+                        answer_code text NOT NULL,
+                        answer_description text,
+                        hint text,
+                        max_submissions integer NOT NULL,
+                        credit text,
+                        data_files text,
+                        back_end text NOT NULL,
+                        expected_text_output text NOT NULL,
+                        expected_image_output text NOT NULL,
+                        instructions text NOT NULL,
+                        output_type text NOT NULL,
+                        show_answer integer NOT NULL,
+                        show_student_submissions integer NOT NULL,
+                        show_expected integer NOT NULL,
+                        show_test_code text,
+                        starter_code text,
+                        test_code text,
+                        date_created timestamp NOT NULL,
+                        date_updated timestamp NOT NULL, enable_pair_programming integer NOT NULL DEFAULT 0, check_code text DEFAULT "",
+                        FOREIGN KEY (course_id) REFERENCES courses (course_id) ON DELETE CASCADE ON UPDATE CASCADE,
+                        FOREIGN KEY (assignment_id) REFERENCES assignments (assignment_id) ON DELETE CASCADE ON UPDATE CASCADE);
+
+INSERT INTO exercises2
+  SELECT *
+  FROM exercises;
+
+DROP TABLE exercises;
+
+ALTER TABLE exercises2 RENAME TO exercises;

front_end/migration_scripts/migrate.py

@@ -20,7 +20,7 @@
 migrate_file_path = f"/migration_scripts/{migration_numbers}_migrate.sql"
 
 settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-content = Content(settings_dict)
+content = ContentSQLite(settings_dict)
 
 check_sql = read_file(check_file_path)
 

front_end/scheduled_scripts/summarize_logs.py

@@ -8,7 +8,7 @@
 from content import *
 
 settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
-content = Content(settings_dict)
+content = ContentSQLite(settings_dict)
 
 #print("Debugging:")
 #print(sys.argv)

front_end/webserver.py

@@ -140,7 +140,8 @@ async def get(self, file_name):
             "secret": secrets_dict["google_oauth_secret"]}
         settings_dict = load_yaml_dict(read_file("/Settings.yaml"))
 
-        content = Content(settings_dict)
+        content = ContentSQLite(settings_dict)
+        content.create_database_tables()
 
         database_version = content.get_database_version()
         code_version = int(read_file("VERSION").rstrip())
@@ -162,7 +163,8 @@ async def get(self, file_name):
                 print("Database migration not needed.")
             elif "***Success***" in result:
                 print(f"Database successfully migrated to version {v+1}.")
-                content.update_database_version(code_version)
+                new_version = v + 1
+                content.update_database_version(new_version)
             else:
                 print(f"Database migration failed for verson {v+1}, so rolling back...")
                 print(result)