From 6e3b32547df413647ad3721ca5d7a8bf4ec09e2d Mon Sep 17 00:00:00 2001 From: Nicole Date: Wed, 18 Dec 2019 15:39:14 +0000 Subject: [PATCH 1/9] Initial commit --- jupyternb/credentials.csv | 2 + jupyternb/jnbchecks.ipynb | 1085 +++++++++++++++++++++++++++++++++++++ jupyternb/jupyterrun.sh | 57 ++ 3 files changed, 1144 insertions(+) create mode 100644 jupyternb/credentials.csv create mode 100644 jupyternb/jnbchecks.ipynb create mode 100644 jupyternb/jupyterrun.sh diff --git a/jupyternb/credentials.csv b/jupyternb/credentials.csv new file mode 100644 index 00000000..ff31614b --- /dev/null +++ b/jupyternb/credentials.csv @@ -0,0 +1,2 @@ +host,username,password +localhost,admin,admin diff --git a/jupyternb/jnbchecks.ipynb b/jupyternb/jnbchecks.ipynb new file mode 100644 index 00000000..0aa03490 --- /dev/null +++ b/jupyternb/jnbchecks.ipynb @@ -0,0 +1,1085 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Process Checks \n", + " \n", + " This notebook will run default process checks on Tickerplant, RDB and HDB. \n", + " Processes are queried via qconnection with the username and password supplied in the credentials.csv file. \n", + " Port numbers are supplied from `. torq.sh summary` table. \n", + "- - - -" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "The raw code for this IPython notebook is by default hidden for easier reading.\n", + "To toggle on/off the raw code, click here." + ], + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import HTML\n", + "HTML('''\n", + "The raw code for this IPython notebook is by default hidden for easier reading.\n", + "To toggle on/off the raw code, click here.''')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tickerplant Results" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Credentials required are host, username, password\n", + "\t host:localhost username: admin password: admin.\n", + "Processed 2 lines.\n" + ] + } + ], + "source": [ + "#importing required modules\n", + "from qpython.qconnection import QConnection as qcon\n", + "from qpython.qcollection import QDictionary as qdict\n", + "from contextlib import redirect_stdout as rd_so\n", + "from datetime import datetime\n", + "from datetime import date\n", + "import time\n", + "import psutil\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "#set qcon variables\n", + "import csv\n", + "with open('credentials.csv') as csv_file:\n", + " csv_reader = csv.reader(csv_file, delimiter=',')\n", + " line_count = 0\n", + " for row in csv_reader:\n", + " if line_count == 0:\n", + " print(f'Credentials required are {\", \".join(row)}')\n", + " line_count += 1\n", + " else:\n", + " host=row[0]\n", + " un=row[1]\n", + " pswd=row[2]\n", + " print(f'\\t host:{row[0]} username: {row[1]} password: {row[2]}.')\n", + " line_count += 1\n", + " print(f'Processed {line_count} lines.') " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Process Summary\n", + "\n", + "Table showing process name, status, PID, Port numbers, CPU and Memory usage.\n", + "* Process status indicated by colours green (up) and red (down).\n", + "* Killtick, tpreplay1 and compression1 should usually have a down status indicated." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TIME PROCESS STATUS PID PORT %MEM %CPU
114:56:03discovery1up265971010.40
214:56:03tickerplant1up275171000.30
314:56:03rdb1up284171025.10
414:56:03hdb1up293371030.51
514:56:03hdb2up302571040.50
614:56:03wdb1up311771050.40
714:56:03sort1up321171060.40
814:56:03gateway1up330371070.40
914:56:03killtickdownnanNone00
1014:56:03monitor1up339571090.40
1114:56:03tpreplay1downnanNone00
1214:56:03housekeeping1up348771110.41
1314:56:03reporter1up357971120.40
1414:56:03compression1downnanNone00
1514:56:04feed1up3761711410
1614:56:04chainedtp1up385371150.30
1714:56:04sortslave1up394371160.30
1814:56:04sortslave2up403571170.30
1914:56:04metrics1up412771180.80
2014:56:04iexfeed1up421971190.40
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#run torq summary\n", + "torq_sum= !\"${TORQHOME}\"/torq.sh summary\n", + "\n", + "#convert array to numpy array\n", + "summary=np.asarray(torq_sum)\n", + "\n", + "#split each element of array by | character\n", + "sum_list = [i.split('|') for i in summary]\n", + "df = pd.DataFrame(sum_list)\n", + "\n", + "# take first row and use as header for df\n", + "new_header = df.iloc[0]\n", + "df = df[1:]\n", + "df.columns = new_header\n", + "\n", + "#trim whitespace from headers\n", + "cols=[]\n", + "for i in df.columns:\n", + " cols.append(str.strip(i))\n", + "df.columns=cols\n", + "\n", + "#trim whitespace from all objects within dataframe\n", + "data = df.select_dtypes(['object'])\n", + "df[data.columns] = df.apply(lambda x: x.str.strip())\n", + "\n", + "#function to extract status of processes - takes a string argument\n", + "def stat_proc(process):\n", + " process_info = df.loc[df['PROCESS'] == process]\n", + " STAT = process_info['STATUS'].astype(str)\n", + " return STAT\n", + "\n", + "# function to extract the port number for each process - takes a string argument\n", + "def find_portno(process):\n", + " process_info = df.loc[df['PROCESS'] == process]\n", + " PORT = process_info['PORT'].astype(str).astype(int)\n", + " return PORT\n", + "\n", + "# function to extract the port number for each process - takes a string argument\n", + "def find_pid(process):\n", + " process_info = df.loc[df['PROCESS'] == process]\n", + " PID = process_info['PID'].astype(str).astype(int)\n", + " return PID\n", + "\n", + "#function to decode bytes to strings\n", + "def byte_decode(table,cols):\n", + " table[cols] = table[cols].applymap(lambda x: x.decode('utf-8'))\n", + "\n", + "#function to print mem and cpu stats\n", + "def print_mem_stats(pid):\n", + " with open(\"tmp.txt\",\"a\") as file:\n", + " with rd_so(file):\n", + " return (psutil.Process(pid)).memory_percent()\n", + "def print_cpu_stats(pid):\n", + " with open(\"tmp.txt\",\"a\") as file:\n", + " with rd_so(file):\n", + " return (psutil.Process(pid)).cpu_percent(interval =1.0)\n", + "\n", + "if stat_proc(\"tickerplant1\").tolist() == [\"down\"]:\n", + " print(\"Please check if the following processes are all up: Tickerplant1, rdb1, hdb1\")\n", + "else:\n", + " #return PIDs\n", + " df.replace('', np.nan, inplace=True)\n", + " pids = df[\"PID\"]\n", + " pids = pids.dropna()\n", + " pids = [int(i) for i in pids]\n", + " \n", + " #create array of mem and cpu stats \n", + " mems=[]\n", + " cpus=[]\n", + " \n", + " for pid in pids:\n", + " mems.append(print_mem_stats(pid));\n", + " cpus.append(print_cpu_stats(pid))\n", + " \n", + " #insert 0 into cpu/mem for processes that are down \n", + " nopid = df[df['PID'].isnull()].index.tolist()\n", + " for i in range(len(nopid)):\n", + " mems.insert((nopid[i]-1),0)\n", + " cpus.insert((nopid[i]-1),0)\n", + " \n", + " #append mem and cpu onto summary table \n", + " df['%MEM']=mems\n", + " df['%CPU']=cpus\n", + " df=df.round({'%MEM':1})\n", + " \n", + " \n", + "# set colour on down processes to red and up processes to green\n", + "def colour_down_red(col):\n", + " color = 'red' if 'down' in col else 'green'\n", + " return 'color: %s' % color\n", + "df.style.applymap(colour_down_red, subset=['STATUS'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Count of tables in Tickerplant \n", + "\n", + "Table to show the count in each table found in the Tickerplant \n", + "* The counts in each of these tables should be 0, as the Tickerplant should not be storing any data.\n", + "* If the counts in any of these tables is not 0, this could indicate a slow subscriber." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logmsgquotequote_iextradetrade_iex
000000
\n", + "
" + ], + "text/plain": [ + " logmsg quote quote_iex trade trade_iex\n", + "0 0 0 0 0 0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if stat_proc(\"tickerplant1\").tolist() == [\"up\"]:\n", + " with qcon(host, port=find_portno('tickerplant1'), username=un, password=pswd,timeout=3.0) as q:\n", + " #counts tickerplant tables\n", + " tablecounts = q(\"enlist tables[]!count each value each tables[]\", pandas=True)\n", + " display(tablecounts)\n", + "else:\n", + " print('The Tickerplant process is down. Unable to count tables in Tickerplant')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tickerplant Log file size increasing\n", + "\n", + "Checks if log messages in the log file of the tickerplant is increasing.\n", + " * If log messages are increasing, the tickerplant is receiving data.\n", + " * If log messages are not increasing, the tickerplant may not be recieving data. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Log file sizes are increasing: True\n" + ] + } + ], + "source": [ + "if stat_proc(\"tickerplant1\").tolist() == [\"up\"]:\n", + " with qcon(host, port=find_portno('tickerplant1'), username=un, password=pswd,timeout=3.0) as q:\n", + " #counts log file size is increasing \n", + " log1=(q(\"hcount .u.L\"))\n", + " time.sleep(2)\n", + " log2=(q(\"hcount .u.L\"))\n", + " print (\"Log file sizes are increasing: \", log1\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
output queueprocesses
handles
\n", + "" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [output queue, processes]\n", + "Index: []" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if stat_proc(\"tickerplant1\").tolist() == [\"up\"]:\n", + " with qcon(host, port=find_portno('tickerplant1'), username=un, password=pswd,timeout=3.0) as q:\n", + " # create pandas dataframe for handles connected to tickerplant\n", + " zW=q(\".z.W[]\", pandas=True)\n", + " hprocesses=q(\"asc select w,u from .clients.clients\", pandas=True)\n", + " byte_decode(hprocesses, ['u'])\n", + " # assign columns with new names\n", + " keys=pd.DataFrame(zW.keys, columns=['handles'])\n", + " values=pd.DataFrame(zW.values, columns=['output queue'])\n", + " hprocesses=hprocesses.rename(columns={'u':'processes'})\n", + " # apply new names\n", + " zW2=keys.join(values)\n", + " # join zW2 and hprocesses\n", + " zW2=zW2.join(hprocesses.processes)\n", + " zW2.set_index('handles', inplace=True)\n", + " # Show only table with slow subscribers \n", + " display(zW2.loc[zW2['output queue'] > 0])\n", + "else:\n", + " print('The Tickerplant process is down. Unable to check process handles connected to Tickerplant.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RDB Results" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "if stat_proc(\"rdb1\").tolist() == [\"up\"]:\n", + " with qcon(host, port=find_portno('rdb1'), username=un, password=pswd,timeout=3.0) as q:\n", + " #Check tables in rdb are same as tables in tickerplant \n", + " tables = q('all 1_tables[] in ((exec w from .servers.SERVERS where proctype=`tickerplant)0)(\"tables[]\")')\n", + " #Check count of tables in rdb - data is being sent from the tickerplant \n", + " tptordb = q('enlist tables[]!count each value each tables[]', pandas=True)\n", + " time.sleep(3)\n", + " tptordb2 = q('enlist tables[]!count each value each tables[]', pandas=True)\n", + " #Check that data in table can be queried\n", + " rdbtquery=q('5#select from last tables[]', pandas=True)\n", + " byte_decode(rdbtquery,['sym'])\n", + " tabname=q('last tables[]')\n", + " #Check that only data from today is present in the rdb tables\n", + " onedatet = q('enlist tables[]!{last exec distinct time.date from x}each tables[]', pandas=True)\n", + "else:\n", + " print('The RDB process is down.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RDB tables\n", + "\n", + "Checks if the tables in the rdb are the same as the tables in the Tickerplant." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RDB tables are same as Tickerplant tables : True\n" + ] + } + ], + "source": [ + "if stat_proc(\"rdb1\").tolist() == [\"up\"]:\n", + " print (\"RDB tables are same as Tickerplant tables : \", tables)\n", + "else:\n", + " print('Unable to check if tables in RDB are the same as Tickerplant tables.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RDB table counts\n", + "\n", + "Checks to see if data is being sent from the Tickerplant to the RDB\n", + "* Indicates whether counts in RDB tables are increasing over time (3 second period)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
heartbeat logmsg quote quote_iex trade trade_iex
0FalseFalseTrueFalseTrueFalse
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if stat_proc(\"rdb1\").tolist() == [\"up\"]:\n", + " tptordb.set_index(tptordb.columns.tolist())\n", + " tptordb2.set_index(tptordb2.columns.tolist())\n", + " rdb_comp = tptordb < tptordb2\n", + " # set colour; True = Green; False = Red\n", + " def rdbtable_colour(val):\n", + " color = 'IndianRed' if val ==False else 'DarkSeaGreen'\n", + " return 'background-color: %s' % color\n", + " display(rdb_comp.style.applymap(rdbtable_colour))\n", + "else:\n", + " print('Unable to count RDB tables.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RDB Query\n", + "Checks to see if tables in the RDB can be queried " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Table to be queried:\n", + "trade_iex\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timesympricesizestopcondexsrctime
0NaTNaNNaNFalseNaT
1NaTNaNNaNFalseNaT
2NaTNaNNaNFalseNaT
3NaTNaNNaNFalseNaT
4NaTNaNNaNFalseNaT
\n", + "
" + ], + "text/plain": [ + " time sym price size stop cond ex srctime\n", + "0 NaT NaN NaN False NaT\n", + "1 NaT NaN NaN False NaT\n", + "2 NaT NaN NaN False NaT\n", + "3 NaT NaN NaN False NaT\n", + "4 NaT NaN NaN False NaT" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if stat_proc(\"rdb1\").tolist() == [\"up\"]:\n", + " print ('Table to be queried:')\n", + " name = tabname.decode('UTF-8')\n", + " print (name)\n", + " display(rdbtquery)\n", + "else:\n", + " print('Unable to query RDB tables.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RDB Date Checks\n", + "Check to see what what date(s) are in tables \n", + "* Green indicates that only today's date is in the RDB table\n", + "* Red indicates there may be more than one date or no date found in the table" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
heartbeat logmsg quote quote_iex trade trade_iex
0No dateNo date2019-12-18 00:00:00No date2019-12-18 00:00:00No date
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if stat_proc(\"rdb1\").tolist() == [\"up\"]:\n", + " onedatet_col=onedatet.fillna(\"No date\")\n", + " def rdbtable_colour(val):\n", + " color = 'IndianRed' if val != date.today() else 'DarkSeaGreen'\n", + " return 'background-color: %s' % color\n", + " display(onedatet_col.style.applymap(rdbtable_colour))\n", + "else:\n", + " print('Unable to check date in RDB tables.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# HDB Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "if stat_proc(\"hdb1\").tolist() == [\"up\"]:\n", + " with qcon(host, port=find_portno('hdb1'), username=un, password=pswd,timeout=10.0) as q:\n", + " #Check hdb table counts excl. eod_summary and eod_summary_iex\n", + " hdbtablecount=q('raze{0!select table:x,cnt:count i by date from x where date >=.z.d-5}each tables[] except `heartbeat`logmsg', pandas=True)\n", + "else:\n", + " print('The HDB process is down.')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### HDB Table Counts\n", + "* Counts for heartbeat and logmsg should be 0\n", + "* If counts for other tables are 0, no data has been recieved for the day before" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "There is no data in the HDB\n" + ] + } + ], + "source": [ + "if stat_proc(\"hdb1\").tolist() == [\"up\"]:\n", + " if isinstance(hdbtablecount, pd.DataFrame):\n", + " byte_decode(hdbtablecount,['table'])\n", + " display(hdbtablecount.set_index(hdbtablecount.columns.tolist()))\n", + " else:\n", + " print('There is no data in the HDB')\n", + "else:\n", + " print('Unable to count HDB tables.')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/jupyternb/jupyterrun.sh b/jupyternb/jupyterrun.sh new file mode 100644 index 00000000..35a1fc55 --- /dev/null +++ b/jupyternb/jupyterrun.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +#must source setenv.sh from TORQHOME as setenv uses dirpath +cd .. +source setenv.sh +#check TORQHOME is set correctly - should point to /path/to/TorQ +echo $TORQHOME + +#set TORQJUPYTER variable path +TORQJUPYTER=${TORQHOME}/jupyternb + +#set JNBCRONSCRIPT variable +JNBCRONSCRIPT=${TORQJUPYTER}/jnbcronjob.sh + +#if JNBCRONSCRIPT does not exist, create empty file +if [[ ! -f ${JNBCRONSCRIPT} ]];then + touch ${JNBCRONSCRIPT} +fi + +#set jupyter notebook variable +JNOTEBOOK=${TORQJUPYTER}/jnbchecks.ipynb + +#set user email variable +JUPYTEREMAIL="putyouremailhere@example.com" + +#set variable for jupyter-nbconvert command path +JUPYTERLOC=/home/$USER/.local/bin/jupyter-nbconvert + +#set variable for HTML version of jupyter notebook +JUPYTERHTML=${TORQJUPYTER}/jnbchecks.html + +#Create script to be run by cron job +#this generates the commands to execute the notebook, +#convert to html and mail to the required user email +echo "#!/bin/bash + +#Check TORQHOME is set in the correct directory +echo $TORQHOME + +#Convert jupyter notebook to HTML version +${JUPYTERLOC} --execute --to html ${JNOTEBOOK} + +#Email HTML version of notebook to user email +echo \"New Jupyter Notebook Generated\" | mail -A ${JUPYTERHTML} -s \"New Jupyter Notebook Generated\" ${JUPYTEREMAIL} + +#Remove most recent notebook, so previous notebooks are not sent +rm ${JUPYTERHTML}" > ${JNBCRONSCRIPT} + +#generate the line to be run in crontab itself +if crontab -l | grep -q 'jnbcron';then + echo "Crontab exists" +else + echo "Crontab does not exist. Creating crontab..." + (crontab -l; echo -e " BASH=/bin/bash\n TORQHOME=${TORQHOME}\n */5 * * * * cd ${TORQHOME}; bash ${JNBCRONSCRIPT}") | crontab - + +fi + From 40e58f6820c011a1f4cd83d75feaa9dadfdac19c Mon Sep 17 00:00:00 2001 From: Nicole Date: Fri, 20 Dec 2019 10:58:09 +0000 Subject: [PATCH 2/9] Emptied output cells of jupyter notebook --- jupyternb/jnbchecks.ipynb | 644 ++------------------------------------ 1 file changed, 24 insertions(+), 620 deletions(-) diff --git a/jupyternb/jnbchecks.ipynb b/jupyternb/jnbchecks.ipynb index 0aa03490..fc17458d 100644 --- a/jupyternb/jnbchecks.ipynb +++ b/jupyternb/jnbchecks.ipynb @@ -14,36 +14,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "The raw code for this IPython notebook is by default hidden for easier reading.\n", - "To toggle on/off the raw code, click here." - ], - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from IPython.display import HTML\n", "HTML('''