diff --git a/README.md b/README.md index 4ccb14f..22215ee 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ A wiki for phpBB with version history, side-by-side compare, and a moderation qu - Tables, written as HTML in an article: ``, ``, ``, ``, ``, `
`, `
` and ``, with `colspan` and `rowspan` on a cell. BBCode and `[[links]]` work inside cells. Any other attribute (`style`, `onclick`, `class`) is dropped, and any other element (`script`, `iframe`) stays plain text. Like headings, tables work in wiki articles only; in a forum post `` stays plain text. After upgrading, disable and enable the extension so the board rebuilds its cache. - Templates: `{{Name}}`, `{{Name|first|second}}` and `{{Name|key=value}}` in an article include the live version of the article `Template:Name`, so a notice or infobox is written once and edited in one place. See [Templates](#templates). - Categories: `[[Category:Name]]` in an article puts it in a category. The article lists its categories at the bottom, `/wiki/Category:Name` lists the articles in a category, and `/wiki/categories` lists every category. See [Categories](#categories). +- Search: `/wiki/search` and a search box on the overview find articles by their title, description and text, with highlighted snippets. It has its own index, kept current as articles are saved, approved, set offline and deleted, and three backends to choose from in the ACP (native, MySQL fulltext, PostgreSQL fulltext). See [Search](#search). - The Wiki link in the board header shows the Wikipedia icon by default. Under **Extensions > Wiki > Settings** in the ACP the icon can be changed to any [Font Awesome](https://fontawesome.com/v4.7.0/icons/) icon name, or left blank for no icon. - A **Start discussion** button on each article creates its discussion topic in a forum chosen under **Extensions > Wiki > Settings** in the ACP, and links the topic to the article. It shows to members who hold the "Can change the discussion topic for an article" permission and may post in that forum, and stays hidden until a forum is chosen. - Separate permissions for viewing, editing, version history, deleting and moderating. @@ -110,6 +111,27 @@ Write `[[Category:Name]]` anywhere in a wiki article to put it in a category. Th - A template page counts for the categories it names outside ``, so `[[Category:Infobox templates]]` puts the template in that category. A category added inside `` by a template shows under the articles that use the template, but those articles are not in the category's list, which is found from the articles' own text. - After upgrading, disable and enable the extension so the board rebuilds its cache. Articles saved before this build keep `[[Category:Name]]` as text until they are saved again or reparsed with `php bin/phpbbcli.php reparser:reparse phpbbmodders_wiki_article`. +## Search + +The wiki has its own search, separate from phpBB's search of posts: **Search the wiki** on the overview, or `/wiki/search?keywords=...`. + +- It searches the live version of each article: its title, its description and its text, with the markup (BBCode, links, templates, tables, categories) reduced to the words in it. Older versions and edits waiting for approval are not searched. Template and category pages are articles too, so they are found. +- Choose to match all the words or any of them, to search titles only or text only, and to sort by last edit or title. Results show the title, description, a snippet with the words marked, and the date. +- It follows the articles: saving a live article, approving a version, setting an article offline, putting it back and deleting it all update the search at once. +- The search needs the permission to read the wiki and phpBB's own "Can use search" permission, and it is off when the board's search is off. + +**In the ACP**, under **Extensions > Wiki > Search**, switch the wiki search on or off, set the results per page, and choose the backend: + +| Backend | For | Notes | +| --- | --- | --- | +| Native | Every database, including SQLite | Keeps its own word tables. Words in most articles are treated as common once a wiki has 100 articles or more, as in phpBB's own search | +| MySQL fulltext | MySQL | Adds fulltext indexes to the search's text table | +| PostgreSQL fulltext | PostgreSQL | Adds text-search indexes to the search's text table | + +Only the backends the database can run are offered. After installing, or after switching backend, press **Create index** under **Search index** for the backend you chose; until then the native backend finds only articles saved since it was installed, and the fulltext backends find nothing. The page shows each index's statistics and can delete it. + +The search keeps a plain-text copy of each live article in its own table (`wiki_search_text`); **Create index** makes that copy again from the articles before it builds the backend's index, so it is also the way to rebuild everything. + ## Linking between articles Write an article's name in double square brackets to link to it, in wiki articles and in forum posts: diff --git a/acp/search_info.php b/acp/search_info.php new file mode 100644 index 0000000..cc4daec --- /dev/null +++ b/acp/search_info.php @@ -0,0 +1,32 @@ + '\phpbbmodders\wiki\acp\search_module', + 'title' => 'ACP_WIKI_SEARCH', + 'modes' => array( + 'settings' => array( + 'title' => 'ACP_WIKI_SEARCH', + 'auth' => 'ext_phpbbmodders/wiki && acl_a_board', + 'cat' => array('ACP_WIKI_TITLE'), + ), + ), + ); + } +} diff --git a/acp/search_module.php b/acp/search_module.php new file mode 100644 index 0000000..87af3bd --- /dev/null +++ b/acp/search_module.php @@ -0,0 +1,50 @@ +get('language'); + $language->add_lang('acp/search'); + $language->add_lang('acp_wiki', 'phpbbmodders/wiki'); + + if (!function_exists('adm_back_link')) + { + include($phpbb_root_path . 'includes/functions_admin.' . $phpEx); + } + + $controller = $phpbb_container->get('phpbbmodders.wiki.acp_search'); + $controller->set_page_url($this->u_action); + + $this->tpl_name = 'acp_wiki_search'; + $this->page_title = $language->lang('ACP_WIKI_SEARCH'); + add_form_key('phpbbmodders_wiki_search'); + + $controller->settings($id, $mode); + $controller->index($id, $mode); + } +} diff --git a/adm/style/acp_wiki_search.html b/adm/style/acp_wiki_search.html new file mode 100644 index 0000000..ea3835c --- /dev/null +++ b/adm/style/acp_wiki_search.html @@ -0,0 +1,100 @@ + + +

{L_ACP_WIKI_SEARCH}

+

{L_WIKI_SEARCH_SETTINGS_EXPLAIN}

+ +
+
+ {L_GENERAL_SEARCH_SETTINGS} +
+

{L_WIKI_SEARCH_ENABLE_EXPLAIN}
+
+ + +
+
+
+

{L_WIKI_PER_PAGE_SEARCH_EXPLAIN}
+
+
+
+
+ {L_SEARCH_TYPE} +
+

{L_SEARCH_TYPE_EXPLAIN}
+
+
+
+ +
+ {L_SUBMIT} +   + +
+ {S_FORM_TOKEN} + + + +

{L_ACP_SEARCH_INDEX}

+ +

{L_CONTINUE_EXPLAIN}

+
+
+ {L_SUBMIT} +   + +
+ {S_FORM_TOKEN} + + + + +

{L_WIKI_SEARCH_INDEX_EXPLAIN}

+ +
+
+ {backend.S_HIDDEN_FIELDS} + {L_INDEX_STATS}{L_COLON} {backend.NAME} ({L_ACTIVE}) + +
+ + + + + + + + + + + + + + + +
{backend.NAME} ({L_ACTIVE})
{L_STATISTIC}{L_VALUE}
{backend.stats.NAME}{L_COLON}{backend.stats.VALUE}
+ +

+ + + + + + + +

+ {S_FORM_TOKEN} + + + + + + diff --git a/config/routing.yml b/config/routing.yml index 979873f..69a0fb6 100644 --- a/config/routing.yml +++ b/config/routing.yml @@ -12,6 +12,10 @@ phpbbmodders_wiki_recent_changes: path: /wiki/recent-changes defaults: { _controller: phpbbmodders.wiki.recent_changes:page } +phpbbmodders_wiki_search: + path: /wiki/search + defaults: { _controller: phpbbmodders.wiki.search:main } + phpbbmodders_wiki_categories: path: /wiki/categories defaults: { _controller: phpbbmodders.wiki.category:index } diff --git a/config/services.yml b/config/services.yml index 086375f..04fc9db 100644 --- a/config/services.yml +++ b/config/services.yml @@ -2,6 +2,10 @@ parameters: tables.wiki.article: '%core.table_prefix%wiki_article' tables.wiki.file: '%core.table_prefix%wiki_file' tables.wiki.watch: '%core.table_prefix%wiki_watch' + tables.wiki.search_text: '%core.table_prefix%wiki_search_text' + tables.wiki.search_results: '%core.table_prefix%wiki_search_results' + tables.wiki.search_wordlist: '%core.table_prefix%wiki_src_wrdlist' + tables.wiki.search_wordmatch: '%core.table_prefix%wiki_src_wrdmtch' services: _defaults: @@ -164,6 +168,7 @@ services: - '%core.php_ext%' - '@phpbbmodders.wiki.watch' - '@phpbbmodders.wiki.templates' + - '@phpbbmodders.wiki.search_sync' phpbbmodders.wiki.compare: class: phpbbmodders\wiki\wiki\compare arguments: @@ -189,6 +194,7 @@ services: - '@phpbbmodders.wiki.files' - '@request' - '@phpbbmodders.wiki.watch' + - '@phpbbmodders.wiki.search_sync' phpbbmodders.wiki.view: class: phpbbmodders\wiki\wiki\view arguments: @@ -274,3 +280,102 @@ services: - [set_watch, ['@phpbbmodders.wiki.watch']] tags: - { name: notification.type } + + # Search: the three backends, the factory that picks the active one, and the + # service that keeps the search's copy of the articles current + phpbbmodders.wiki.search.fulltext.mysql: + class: phpbbmodders\wiki\search\backend\wiki_fulltext_mysql + arguments: + - '@config' + - '@dbal.conn' + - '@language' + - '@user' + - '%tables.wiki.search_text%' + - '%tables.wiki.search_results%' + - '%core.root_path%' + - '%core.php_ext%' + tags: + - { name: phpbbmodders.wiki.search.backend } + + phpbbmodders.wiki.search.fulltext.native: + class: phpbbmodders\wiki\search\backend\wiki_fulltext_native + arguments: + - '@config' + - '@dbal.conn' + - '@language' + - '@user' + - '%tables.wiki.search_text%' + - '%tables.wiki.search_results%' + - '%tables.wiki.search_wordmatch%' + - '%tables.wiki.search_wordlist%' + - '%core.root_path%' + - '%core.php_ext%' + tags: + - { name: phpbbmodders.wiki.search.backend } + + phpbbmodders.wiki.search.fulltext.postgres: + class: phpbbmodders\wiki\search\backend\wiki_fulltext_postgres + arguments: + - '@config' + - '@dbal.conn' + - '@language' + - '@user' + - '%tables.wiki.search_text%' + - '%tables.wiki.search_results%' + - '%core.root_path%' + - '%core.php_ext%' + tags: + - { name: phpbbmodders.wiki.search.backend } + + phpbbmodders.wiki.search.backend_collection: + class: phpbb\di\service_collection + arguments: + - '@service_container' + tags: + - { name: service_collection, tag: phpbbmodders.wiki.search.backend, class_name_aware: true } + + phpbbmodders.wiki.search.backend_factory: + class: phpbbmodders\wiki\search\wiki_search_backend_factory + arguments: + - '@config' + - '@phpbbmodders.wiki.search.backend_collection' + + phpbbmodders.wiki.search_sync: + class: phpbbmodders\wiki\wiki\search_sync + arguments: + - '@config' + - '@dbal.conn' + - '@phpbbmodders.wiki.search.backend_factory' + - '%tables.wiki.article%' + - '%tables.wiki.search_text%' + + phpbbmodders.wiki.acp_search: + class: phpbbmodders\wiki\controller\acp_search + arguments: + - '@config' + - '@language' + - '@log' + - '@request' + - '@template' + - '@user' + - '@phpbbmodders.wiki.search.backend_collection' + - '@phpbbmodders.wiki.search.backend_factory' + - '@phpbbmodders.wiki.search_sync' + + phpbbmodders.wiki.search: + class: phpbbmodders\wiki\controller\search + arguments: + - '@auth' + - '@config' + - '@dbal.conn' + - '@controller.helper' + - '@language' + - '@pagination' + - '@request' + - '@template' + - '@user' + - '@phpbbmodders.wiki.search.backend_factory' + - '%tables.wiki.article%' + - '%tables.wiki.search_text%' + - '%core.root_path%' + - '%core.php_ext%' diff --git a/controller/acp_search.php b/controller/acp_search.php new file mode 100644 index 0000000..101d06a --- /dev/null +++ b/controller/acp_search.php @@ -0,0 +1,435 @@ +config = $config; + $this->language = $language; + $this->log = $log; + $this->request = $request; + $this->template = $template; + $this->user = $user; + $this->backends = $backends; + $this->factory = $factory; + $this->search_sync = $search_sync; + } + + /** + * @param string $u_action The address of the page + * @return void + */ + public function set_page_url($u_action) + { + $this->u_action = $u_action; + } + + /** + * The settings: search on or off, results to a page, and the backend. + * + * @param string $id Module id + * @param string $mode Module mode + * @return void + */ + public function settings($id, $mode) + { + $submit = $this->request->is_set_post('submit'); + if ($submit && !check_form_key('phpbbmodders_wiki_search')) + { + trigger_error($this->language->lang('FORM_INVALID') . adm_back_link($this->u_action), E_USER_WARNING); + } + + $search_options = ''; + foreach ($this->backends as $search) + { + // Only the backends this database can use + if ($search->is_available()) + { + $type = $search->get_type(); + $selected = ($this->config['wiki_search_type'] === $type) ? ' selected="selected"' : ''; + $search_options .= ''; + } + } + + $cfg_array = $this->request->is_set('config') ? $this->request->variable('config', array('' => ''), true) : array(); + $updated = $this->request->variable('updated', false); + + $settings = array('wiki_search' => 'bool', 'wiki_per_page_search' => 'integer:1:9999'); + foreach ($settings as $config_name => $var_type) + { + if (!isset($cfg_array[$config_name])) + { + continue; + } + + $var_type = explode(':', $var_type); + $config_value = $cfg_array[$config_name]; + settype($config_value, $var_type[0]); + if (isset($var_type[1])) + { + $config_value = max((int) $var_type[1], $config_value); + } + if (isset($var_type[2])) + { + $config_value = min((int) $var_type[2], $config_value); + } + + if ($submit && ($this->config[$config_name] != $config_value)) + { + $this->config->set($config_name, $config_value); + $updated = true; + } + } + + if ($submit) + { + $extra_message = ''; + if ($updated) + { + $this->log->add('admin', $this->user->data['user_id'], $this->user->data['user_ip'], 'LOG_WIKI_CONFIG_SEARCH', time()); + } + + if (isset($cfg_array['wiki_search_type']) && ($cfg_array['wiki_search_type'] != $this->config['wiki_search_type'])) + { + $search = $this->factory->get($cfg_array['wiki_search_type']); + if (confirm_box(true)) + { + // A backend that is not ready gives a message; false means it is + if (!($error = $search->init())) + { + $this->config->set('wiki_search_type', $cfg_array['wiki_search_type']); + $extra_message = '
' . $this->language->lang('SWITCHED_SEARCH_BACKEND'); + } + else + { + trigger_error($error . adm_back_link($this->u_action), E_USER_WARNING); + } + } + else + { + // The confirmed request is checked against the form key above, so the token + // it was first sent with has to travel through the confirm page too + confirm_box(false, $this->language->lang('CONFIRM_SEARCH_BACKEND'), build_hidden_fields(array( + 'i' => $id, + 'mode' => $mode, + 'submit' => true, + 'updated' => $updated, + 'config' => array('wiki_search_type' => $cfg_array['wiki_search_type']), + 'creation_time' => $this->request->variable('creation_time', 0), + 'form_token' => $this->request->variable('form_token', ''), + ))); + } + } + + trigger_error($this->language->lang('CONFIG_UPDATED') . $extra_message . adm_back_link($this->u_action)); + } + + $this->template->assign_vars(array( + 'S_SEARCH_TYPES' => $search_options, + 'S_WIKI_SEARCH' => (bool) $this->config['wiki_search'], + 'WIKI_PER_PAGE_SEARCH' => $this->config['wiki_per_page_search'] ? (int) $this->config['wiki_per_page_search'] : 10, + 'S_SETTINGS' => true, + 'U_ACTION' => $this->u_action . '&hash=' . generate_link_hash('phpbbmodders_wiki_search'), + )); + } + + /** + * Run what the index part of the page asks for: show the backends, continue a run, or + * start or stop one. + * + * @param string $id Module id + * @param string $mode Module mode + * @return void + */ + public function index($id, $mode) + { + $action = $this->request->variable('action', ''); + $state = !empty($this->config['wiki_search_indexing_state']) ? explode(',', $this->config['wiki_search_indexing_state']) : array(); + + if ($action && !$this->request->is_set_post('cancel')) + { + switch ($action) + { + case 'progress_bar': + $this->display_progress_bar(); + break; + + case 'create': + case 'delete': + if ($this->request->is_set_post('action') || $this->request->is_set_post('continue')) + { + $this->require_form_key(); + } + else if (empty($state) || $state[self::STATE_ACTION] !== $action || !check_link_hash($this->request->variable('hash', ''), 'phpbbmodders_wiki_search_' . $action)) + { + trigger_error($this->language->lang('FORM_INVALID') . adm_back_link($this->u_action), E_USER_WARNING); + } + $this->index_action($id, $mode, $action, $state); + break; + + default: + trigger_error('NO_ACTION', E_USER_ERROR); + } + + return; + } + + if ($this->request->is_set_post('cancel')) + { + $this->require_form_key(); + $state = array(); + $this->save_state($state); + } + + if (!empty($state)) + { + $this->index_inprogress($state[self::STATE_ACTION]); + } + else + { + $this->index_overview(); + } + } + + /** + * List the backends with their statistics and the button to build or delete the index. + * + * @return void + */ + protected function index_overview() + { + foreach ($this->backends as $search) + { + // A backend this database cannot use has no index to build + if (!$search->is_available()) + { + continue; + } + + $this->template->assign_block_vars('backend', array( + 'NAME' => $search->get_name(), + 'TYPE' => $search->get_type(), + 'S_ACTIVE' => $search->get_type() === $this->config['wiki_search_type'], + 'S_HIDDEN_FIELDS' => build_hidden_fields(array('wiki_search_type' => $search->get_type())), + 'S_INDEXED' => $search->index_created(), + )); + + $stats = $search->index_stats(); + if (!empty($stats)) + { + foreach ($stats as $name => $value) + { + $this->template->assign_block_vars('backend.stats', array( + 'NAME' => $name, + 'VALUE' => $value, + )); + } + } + } + + $this->template->assign_vars(array( + 'U_INDEX_ACTION' => $this->u_action, + 'UA_PROGRESS_BAR' => addslashes($this->u_action . '&action=progress_bar'), + )); + } + + /** + * The form to continue or cancel a run that was left half done. + * + * @param string $action 'create' or 'delete' + * @return void + */ + protected function index_inprogress($action) + { + $this->template->assign_vars(array( + 'U_CONTINUE_INDEXING' => $this->u_action . '&action=' . $action, + 'UA_PROGRESS_BAR' => addslashes($this->u_action . '&action=progress_bar'), + 'L_CONTINUE' => ($action === 'create') ? $this->language->lang('CONTINUE_INDEXING') : $this->language->lang('CONTINUE_DELETING_INDEX'), + 'L_CONTINUE_EXPLAIN' => ($action === 'create') ? $this->language->lang('CONTINUE_INDEXING_EXPLAIN') : $this->language->lang('CONTINUE_DELETING_INDEX_EXPLAIN'), + 'S_ACTION' => $action, + 'S_CONTINUE' => true, + )); + } + + /** + * Do a step of building or deleting the index, and keep going until it is done. + * + * @param string $id Module id + * @param string $mode Module mode + * @param string $action 'create' or 'delete' + * @param array $state The saved state, empty on the first step + * @return void + */ + protected function index_action($id, $mode, $action, array $state) + { + ini_set('memory_limit', '128M'); + + if (empty($state)) + { + if (!$this->request->is_set_post('wiki_search_type')) + { + trigger_error($this->language->lang('FORM_INVALID') . adm_back_link($this->u_action), E_USER_WARNING); + } + + $state = array( + self::STATE_SEARCH_TYPE => $this->request->variable('wiki_search_type', ''), + self::STATE_ACTION => $action, + self::STATE_ARTICLE_COUNTER => 0, + ); + $this->save_state($state); + } + + $type = $state[self::STATE_SEARCH_TYPE]; + $action = $state[self::STATE_ACTION]; + $article_counter = &$state[self::STATE_ARTICLE_COUNTER]; + + $search = $this->factory->get($type); + + try + { + if ($action === 'create' && (int) $article_counter === 0) + { + // The backends index the search's plain-text copy of the articles, so make it current first + $this->search_sync->resync_all(); + } + + $status = ($action === 'create') ? $search->create_index($article_counter) : $search->delete_index($article_counter); + if ($status) + { + $this->save_state($state); + + $u_action = append_sid($this->u_action, 'i=' . $id . '&mode=' . $mode . '&action=' . $action . '&hash=' . generate_link_hash('phpbbmodders_wiki_search_' . $action)); + meta_refresh(1, $u_action); + + $message_redirect = $this->language->lang(($action === 'create') ? 'SEARCH_INDEX_CREATE_REDIRECT' : 'SEARCH_INDEX_DELETE_REDIRECT', (int) $status['row_count'], $status['post_counter']); + $message_rate = $this->language->lang(($action === 'create') ? 'SEARCH_INDEX_CREATE_REDIRECT_RATE' : 'SEARCH_INDEX_DELETE_REDIRECT_RATE', $status['rows_per_second']); + trigger_error($message_redirect . $message_rate); + } + } + catch (\Exception $e) + { + $this->save_state(array()); + trigger_error($e->getMessage() . adm_back_link($this->u_action) . $this->close_popup_js(), E_USER_WARNING); + } + + $search->tidy(); + $this->save_state(array()); + + $this->log->add('admin', $this->user->data['user_id'], $this->user->ip, ($action === 'create') ? 'LOG_SEARCH_INDEX_CREATED' : 'LOG_SEARCH_INDEX_REMOVED', false, array($search->get_name())); + + trigger_error($this->language->lang(($action === 'create') ? 'SEARCH_INDEX_CREATED' : 'SEARCH_INDEX_REMOVED') . adm_back_link($this->u_action) . $this->close_popup_js()); + } + + /** + * The small window shown while an index is being built. + * + * @return void + */ + protected function display_progress_bar() + { + $type = $this->request->variable('type', ''); + $l_type = ($type === 'create') ? 'INDEXING_IN_PROGRESS' : 'DELETING_INDEX_IN_PROGRESS'; + + adm_page_header($this->language->lang($l_type)); + $this->template->set_filenames(array('body' => 'progress_bar.html')); + $this->template->assign_vars(array( + 'L_PROGRESS' => $this->language->lang($l_type), + 'L_PROGRESS_EXPLAIN' => $this->language->lang($l_type . '_EXPLAIN'), + )); + adm_page_footer(); + } + + /** + * JavaScript that closes the waiting window; it is attached to the messages. + * + * @return string + */ + protected function close_popup_js() + { + return "\n"; + } + + /** + * @param array $state + * @return void + */ + protected function save_state(array $state) + { + ksort($state); + $this->config->set('wiki_search_indexing_state', implode(',', $state), true); + } + + /** + * Refuse a form that does not carry the page's token. + * + * @return void + */ + protected function require_form_key() + { + if (!check_form_key('phpbbmodders_wiki_search')) + { + trigger_error($this->language->lang('FORM_INVALID') . adm_back_link($this->u_action), E_USER_WARNING); + } + } +} diff --git a/controller/overview.php b/controller/overview.php index f4b6c77..1af9f10 100644 --- a/controller/overview.php +++ b/controller/overview.php @@ -85,6 +85,7 @@ public function base() $this->user->add_lang_ext('phpbbmodders/wiki', 'common'); $this->user->add_lang_ext('phpbbmodders/wiki', 'overview'); + $this->user->add_lang_ext('phpbbmodders/wiki', 'search'); // get all article diff --git a/controller/search.php b/controller/search.php new file mode 100644 index 0000000..0050dc5 --- /dev/null +++ b/controller/search.php @@ -0,0 +1,283 @@ +auth = $auth; + $this->config = $config; + $this->db = $db; + $this->helper = $helper; + $this->language = $language; + $this->pagination = $pagination; + $this->request = $request; + $this->template = $template; + $this->user = $user; + $this->factory = $factory; + $this->article_table = $article_table; + $this->search_text_table = $search_text_table; + $this->phpbb_root_path = $phpbb_root_path; + $this->php_ext = $php_ext; + } + + /** + * Check that the wiki search may be used right now, and stop with a message if not. + * + * @return void + */ + protected function check_allowed() + { + if (!$this->auth->acl_get('u_wiki_view')) + { + trigger_error('NOT_AUTHORISED'); + } + if (!$this->config['wiki_search']) + { + trigger_error('SEARCH_DISABLED'); + } + if (!$this->auth->acl_get('u_search') || !$this->config['load_search']) + { + $this->template->assign_var('S_NO_SEARCH', true); + trigger_error('NO_SEARCH'); + } + if ($this->user->load && $this->config['limit_search_load'] && $this->user->load > (float) $this->config['limit_search_load']) + { + $this->template->assign_var('S_NO_SEARCH', true); + trigger_error('NO_SEARCH_TIME'); + } + } + + /** + * The words to mark in the results, as a regular expression part, from the search words. + * + * @param string $keywords What was typed + * @return array The words, each already safe in a regular expression + */ + protected function highlight_words($keywords) + { + $hilit = phpbb_clean_search_string(str_replace(array('+', '-', '|', '(', ')', '"'), ' ', $keywords)); + $words = array_filter(explode(' ', $hilit), 'strlen'); + + $safe = array(); + foreach ($words as $word) + { + $word = phpbb_clean_search_string($word); + $word = str_replace('\*', '\w*?', preg_quote($word, '#')); + $word = preg_replace('#(^|\s)\\\\w\*\?(\s|$)#', '$1\w+?$2', $word); + if ($word !== '') + { + $safe[] = $word; + } + } + + return $safe; + } + + /** + * The search page. + * + * @return \Symfony\Component\HttpFoundation\Response + */ + public function main() + { + $this->user->add_lang_ext('phpbbmodders/wiki', 'common'); + $this->language->add_lang('search'); + $this->language->add_lang('search', 'phpbbmodders/wiki'); + $this->check_allowed(); + + include_once($this->phpbb_root_path . 'includes/functions_content.' . $this->php_ext); + + $keywords = $this->request->variable('keywords', '', true); + $terms = ($this->request->variable('terms', 'all') === 'any') ? 'any' : 'all'; + $fields = $this->request->variable('sf', 'all'); + $fields = in_array($fields, array('all', 'titleonly', 'msgonly'), true) ? $fields : 'all'; + $sort_key = $this->request->variable('sk', 't'); + $sort_key = in_array($sort_key, array('t', 's'), true) ? $sort_key : 't'; + $sort_dir = ($this->request->variable('sd', 'd') === 'a') ? 'a' : 'd'; + $start = max(0, $this->request->variable('start', 0)); + + $u_search = $this->helper->route('phpbbmodders_wiki_search'); + $this->template->assign_block_vars('navlinks', array( + 'FORUM_NAME' => $this->user->lang('WIKI'), + 'U_VIEW_FORUM' => $this->helper->route('phpbbmodders_wiki_index', array()), + )); + $this->template->assign_block_vars('navlinks', array( + 'FORUM_NAME' => $this->user->lang('WIKI_SEARCH'), + 'U_VIEW_FORUM' => $u_search, + )); + + $search = $this->factory->get_active(); + $this->template->assign_vars(array( + 'KEYWORDS' => $keywords, + 'S_TERMS_ANY' => ($terms === 'any'), + 'S_FIELDS' => $fields, + 'S_SORT_KEY' => $sort_key, + 'S_SORT_DIR' => $sort_dir, + 'U_SEARCH' => $u_search, + 'S_SEARCH_NOT_BUILT' => !$search->index_created(), + 'U_ACP_SEARCH' => $this->auth->acl_get('a_board') ? append_sid($this->phpbb_root_path . 'adm/index.' . $this->php_ext, 'i=-phpbbmodders-wiki-acp-search_module&mode=settings') : '', + )); + + if ($keywords === '') + { + return $this->helper->render('search.html', $this->user->lang('WIKI_SEARCH')); + } + + $correct = $search->split_keywords($keywords, $terms); + if (!$correct || !$search->get_search_query()) + { + $common = $search->get_common_words(); + $ignored = count($common) ? $this->language->lang('IGNORED_TERMS_EXPLAIN', implode(' ', $common)) . '
' : ''; + $length = $search->get_word_length(); + trigger_error($ignored . ($length ? $this->language->lang('NO_KEYWORDS', $this->language->lang('CHARACTERS', (int) $length['min']), $this->language->lang('CHARACTERS', (int) $length['max'])) : '')); + } + + $per_page = max(1, (int) $this->config['wiki_per_page_search']); + $sort_by_sql = array('t' => 'article_date', 's' => 'article_title'); + $ids = array(); + $total = (int) $search->keyword_search('posts', $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, '0', array(), 0, array(), '', $ids, $start, $per_page); + + $words = $this->highlight_words($keywords); + $pattern = !empty($words) ? '#(?!<.*)(?]*(?:)#isu' : ''; + + if (!empty($ids)) + { + $sql = 'SELECT s.article_id, s.article_title, s.article_description, s.article_body, s.article_date, s.author, a.article_url, a.article_user_id + FROM ' . $this->search_text_table . ' s + INNER JOIN ' . $this->article_table . ' a ON (a.article_id = s.article_id) + WHERE ' . $this->db->sql_in_set('s.article_id', $ids) . ' + AND a.article_approved = 1'; + $result = $this->db->sql_query($sql); + $rows = array(); + while ($row = $this->db->sql_fetchrow($result)) + { + $rows[(int) $row['article_id']] = $row; + } + $this->db->sql_freeresult($result); + + // In the order the backend gave + foreach ($ids as $id) + { + if (!isset($rows[(int) $id])) + { + continue; + } + + $row = $rows[(int) $id]; + $title = htmlspecialchars($row['article_title'], ENT_COMPAT, 'UTF-8'); + $description = htmlspecialchars($row['article_description'], ENT_COMPAT, 'UTF-8'); + $snippet = ($fields === 'titleonly') ? '' : get_context(htmlspecialchars($row['article_body'], ENT_COMPAT, 'UTF-8'), $words ? array_map('stripslashes', $words) : array(), self::SNIPPET_LENGTH); + if ($pattern !== '') + { + $mark = '$1'; + $title = preg_replace($pattern, $mark, $title); + $description = preg_replace($pattern, $mark, $description); + $snippet = preg_replace($pattern, $mark, $snippet); + } + + $this->template->assign_block_vars('results', array( + 'TITLE' => $title, + 'DESCRIPTION' => $description, + 'SNIPPET' => $snippet, + 'DATE' => $this->user->format_date($row['article_date']), + 'AUTHOR' => htmlspecialchars($row['author'], ENT_COMPAT, 'UTF-8'), + 'U_ARTICLE' => $this->helper->route('phpbbmodders_wiki_article', array('article' => $row['article_url'])), + )); + } + } + + $u_params = array('keywords' => $keywords); + foreach (array('terms' => $terms, 'sf' => $fields, 'sk' => $sort_key, 'sd' => $sort_dir) as $name => $value) + { + $u_params[$name] = $value; + } + $this->pagination->generate_template_pagination($this->helper->route('phpbbmodders_wiki_search', $u_params), 'pagination', 'start', $total, $per_page, $start); + + $this->template->assign_vars(array( + 'S_RESULTS' => true, + 'TOTAL_MATCHES' => $total, + 'SEARCH_MATCHES' => $this->user->lang('WIKI_SEARCH_FOUND', $total), + 'TOTAL_ITEMS' => $this->user->lang('TOTAL_ITEMS', $total), + 'PAGE_NUMBER' => $this->pagination->on_page($total, $per_page, $start), + )); + + return $this->helper->render('search.html', $this->user->lang('WIKI_SEARCH')); + } +} diff --git a/docs/TODO.md b/docs/TODO.md index 86b39fb..d97184b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -47,21 +47,10 @@ Not built: ## Wiki search -Confirmed: wiki content is **not** in phpBB's board search at all. Checked -both ways — no search-related code anywhere in the extension, and live on -the test board, searching for a word that only exists in a live, approved -article's text ("trivial", from "Getting Started") returns zero matches. -`wiki_article` is a non-standard table outside phpBB's normal post/topic -search index, so this isn't surprising, but it does mean articles are -currently undiscoverable except by browsing the overview page or already -knowing the URL. - -Chosen direction (confirmed 10/12/2026): first port the Knowledge Base's search engine to the -wiki, then show wiki matches in a block above the results of phpBB's normal board search -(grouped results, not one mixed list), with an "Also search the Wiki" checkbox on Advanced -search. Searches respect the wiki's viewing permissions, use only the current approved article text -and need no core change. The full plan, scope and verification list are in -[`TODO/todo-forum-search-integration.md`](TODO/todo-forum-search-integration.md). +Built: the wiki's own search page, index and ACP page (a port of the Knowledge Base's engine; see +the README and [`docs/TODO/todo-forum-search-integration.md`](TODO/todo-forum-search-integration.md)). +Next: a "Wiki (N)" block and an "Also search the Wiki" checkbox in phpBB's own search, with the wiki +results kept apart from the board results. ## A wiki-style UI, not just reskinned forum templates diff --git a/docs/TODO/todo-forum-search-integration.md b/docs/TODO/todo-forum-search-integration.md index 8f05dc0..8369eb5 100644 --- a/docs/TODO/todo-forum-search-integration.md +++ b/docs/TODO/todo-forum-search-integration.md @@ -1,6 +1,6 @@ # Wiki search and forum search integration -Status: direction confirmed 10/12/2026, not started. Companion to the Knowledge Base's plan in +Status: direction confirmed 10/12/2026. Step 1 is built (the wiki's own search); step 2 is not started. Companion to the Knowledge Base's plan in `phpbbmodders/knowledgebase` (`docs/TODO/todo-forum-search-integration.md`). ## Goal @@ -10,7 +10,7 @@ search also sees matching wiki articles beside the forum results ("option 1": gr one mixed list). Today the wiki has no search at all: its articles are undiscoverable except by browsing the overview page or knowing the URL. -## Step 1: give the wiki a search by porting the Knowledge Base's +## Step 1 (built): give the wiki a search by porting the Knowledge Base's The Knowledge Base already has a complete search: its own copies of phpBB's three search engines (native, MySQL fulltext, PostgreSQL fulltext) behind a small interface and a factory, its own index diff --git a/event/listener.php b/event/listener.php index 36668af..e691688 100644 --- a/event/listener.php +++ b/event/listener.php @@ -176,6 +176,7 @@ public function page_header($event) 'U_WIKI_RECENT_CHANGES' => $this->helper->route('phpbbmodders_wiki_recent_changes', array()), 'U_WIKI_TEMPLATES' => $this->helper->route('phpbbmodders_wiki_templates', array()), 'U_WIKI_CATEGORIES' => $this->helper->route('phpbbmodders_wiki_categories', array()), + 'U_WIKI_SEARCH' => !empty($this->config['wiki_search']) ? $this->helper->route('phpbbmodders_wiki_search', array()) : '', 'U_WIKI_FEED_RSS' => \phpbbmodders\wiki\wiki\feed::offers($feed_setting, \phpbbmodders\wiki\wiki\feed::RSS) ? $this->helper->route('phpbbmodders_wiki_recent_changes_feed', array('_format' => 'rss')) : '', 'U_WIKI_FEED_ATOM' => \phpbbmodders\wiki\wiki\feed::offers($feed_setting, \phpbbmodders\wiki\wiki\feed::ATOM) ? $this->helper->route('phpbbmodders_wiki_recent_changes_feed', array('_format' => 'atom')) : '', 'WIKI_ICON_CLASSES' => \phpbbmodders\wiki\wiki\icon::classes(isset($this->config['wiki_icon']) ? $this->config['wiki_icon'] : \phpbbmodders\wiki\wiki\icon::DEFAULT_ICON), diff --git a/language/en/acp_wiki.php b/language/en/acp_wiki.php index 627d73b..6360dca 100644 --- a/language/en/acp_wiki.php +++ b/language/en/acp_wiki.php @@ -34,5 +34,13 @@ 'WIKI_DISCUSSION_FORUM' => 'Discussion forum', 'WIKI_DISCUSSION_FORUM_EXPLAIN' => 'The forum where the “Start discussion” button on an article creates its discussion topic. Choose “None (disabled)” to hide the button.', 'WIKI_DISCUSSION_FORUM_NONE' => '-- None (disabled) --', + 'ACP_WIKI_SEARCH' => 'Search', + 'WIKI_SEARCH_SETTINGS_EXPLAIN' => 'The wiki has its own search. Choose how it finds articles and build its index; the search covers the live version of each article and keeps itself current when articles are saved, approved or deleted.', + 'WIKI_SEARCH_ENABLE' => 'Enable wiki search', + 'WIKI_SEARCH_ENABLE_EXPLAIN' => 'Lets members search the wiki. When off, the search page and the search box are hidden.', + 'WIKI_PER_PAGE_SEARCH' => 'Results per page', + 'WIKI_PER_PAGE_SEARCH_EXPLAIN' => 'How many articles the wiki search results page shows at a time.', + 'WIKI_SEARCH_INDEX_EXPLAIN' => 'Build the index of the search backend you use. Until it is built the native backend finds only articles saved since it was installed, and the MySQL and PostgreSQL backends find nothing. Switching backend does not delete the other one’s index.', + 'LOG_WIKI_CONFIG_SEARCH' => 'Altered wiki search settings', 'WIKI_DISCUSSION_FORUM_INVALID' => 'The forum you chose does not exist or cannot hold topics.', )); diff --git a/language/en/info_acp_wiki.php b/language/en/info_acp_wiki.php index 462c64b..c55889a 100644 --- a/language/en/info_acp_wiki.php +++ b/language/en/info_acp_wiki.php @@ -21,4 +21,5 @@ $lang = array_merge($lang, array( 'ACP_WIKI_TITLE' => 'Wiki', 'ACP_WIKI_SETTINGS' => 'Settings', + 'ACP_WIKI_SEARCH' => 'Search', )); diff --git a/language/en/search.php b/language/en/search.php new file mode 100644 index 0000000..89def9f --- /dev/null +++ b/language/en/search.php @@ -0,0 +1,41 @@ + 'Search the wiki', + 'WIKI_SEARCH_BOX' => 'Search the wiki', + 'WIKI_SEARCH_NOT_BUILT' => 'The wiki search has not been set up yet, so it may find nothing.', + 'WIKI_SEARCH_BUILD' => 'Build the search index', + 'WIKI_SEARCH_IN' => 'Search in', + 'WIKI_SEARCH_IN_ALL' => 'Titles, descriptions and text', + 'WIKI_SEARCH_IN_TITLES' => 'Titles only', + 'WIKI_SEARCH_IN_TEXT' => 'Text only', + 'WIKI_SORT_DATE' => 'Last edited', + 'WIKI_SORT_TITLE' => 'Title', + 'WIKI_SEARCH_NO_RESULTS' => 'No article matches your search.', + 'WIKI_SEARCH_FOUND' => array( + 0 => 'No article found', + 1 => 'Found 1 article', + 2 => 'Found %d articles', + ), +)); diff --git a/migrations/add_wiki_search.php b/migrations/add_wiki_search.php new file mode 100644 index 0000000..9506b95 --- /dev/null +++ b/migrations/add_wiki_search.php @@ -0,0 +1,121 @@ + array( + $this->table_prefix . 'wiki_search_text' => array( + 'COLUMNS' => array( + 'article_id' => array('UINT', 0), + 'article_category_id' => array('UINT', 0), + 'approved' => array('BOOL', 1), + 'article_title' => array('VCHAR:255', ''), + 'article_description' => array('VCHAR:255', ''), + 'article_date' => array('UINT:11', 0), + 'author_id' => array('UINT', 0), + 'author' => array('VCHAR:255', ''), + 'article_body' => array('MTEXT_UNI', ''), + ), + 'PRIMARY_KEY' => 'article_id', + 'KEYS' => array( + 'author_id' => array('INDEX', 'author_id'), + 'author' => array('INDEX', 'author'), + ), + ), + $this->table_prefix . 'wiki_src_wrdlist' => array( + 'COLUMNS' => array( + 'word_id' => array('UINT', null, 'auto_increment'), + 'word_text' => array('VCHAR_UNI', ''), + 'word_common' => array('BOOL', 0), + 'word_count' => array('UINT', 0), + ), + 'PRIMARY_KEY' => 'word_id', + 'KEYS' => array( + 'word_text' => array('UNIQUE', 'word_text'), + 'word_count' => array('INDEX', 'word_count'), + ), + ), + $this->table_prefix . 'wiki_src_wrdmtch' => array( + 'COLUMNS' => array( + 'article_id' => array('UINT', 0), + 'word_id' => array('UINT', 0), + 'title_match' => array('BOOL', 0), + 'descr_match' => array('BOOL', 0), + ), + 'KEYS' => array( + // A word in an article's text and in its description is two rows that differ only in + // descr_match, so that column is part of the key + 'un_mtch' => array('UNIQUE', array('article_id', 'word_id', 'title_match', 'descr_match')), + 'word_id' => array('INDEX', 'word_id'), + 'article_id' => array('INDEX', 'article_id'), + ), + ), + $this->table_prefix . 'wiki_search_results' => array( + 'COLUMNS' => array( + 'search_key' => array('VCHAR:32', ''), + 'search_time' => array('UINT:11', 0), + 'search_keywords' => array('MTEXT_UNI', ''), + 'search_authors' => array('MTEXT_UNI', ''), + ), + 'PRIMARY_KEY' => 'search_key', + ), + ), + ); + } + + public function revert_schema() + { + return array( + 'drop_tables' => array( + $this->table_prefix . 'wiki_search_text', + $this->table_prefix . 'wiki_src_wrdlist', + $this->table_prefix . 'wiki_src_wrdmtch', + $this->table_prefix . 'wiki_search_results', + ), + ); + } + + public function update_data() + { + return array( + array('config.add', array('wiki_search', 1)), + array('config.add', array('wiki_search_type', self::DEFAULT_BACKEND)), + array('config.add', array('wiki_per_page_search', 10)), + array('config.add', array('wiki_num_articles', 0)), + + array('module.add', array('acp', 'ACP_WIKI_TITLE', array( + 'module_basename' => '\phpbbmodders\wiki\acp\search_module', + 'modes' => array('settings'), + ))), + ); + } +} diff --git a/search/backend/wiki_base.php b/search/backend/wiki_base.php new file mode 100644 index 0000000..f7a6196 --- /dev/null +++ b/search/backend/wiki_base.php @@ -0,0 +1,458 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbbmodders\wiki\search\backend; + +use phpbb\cache\service; +use phpbb\config\config; +use phpbb\db\driver\driver_interface; +use phpbb\user; + +/** + * optional base class for search plugins providing simple caching based on ACM + * and functions to retrieve ignore_words and synonyms + */ +abstract class wiki_base +{ + public const WIKI_SEARCH_RESULT_NOT_IN_CACHE = 0; + public const WIKI_SEARCH_RESULT_IN_CACHE = 1; + public const WIKI_SEARCH_RESULT_INCOMPLETE = 2; + + // Batch size for create_index and delete_index + private const BATCH_SIZE = 100; + + /** @var service */ + protected service $cache; + + /** @var config */ + protected config $config; + + /** @var driver_interface */ + protected driver_interface $db; + + /** @var user */ + protected user $user; + + /** @var string */ + protected string $table_prefix; + + /** @var string */ + protected string $search_results_table; + + /** + * Constructor. + * + * @param service $cache + * @param config $config + * @param driver_interface $db + * @param user $user + * @param string $search_results_table + */ + public function __construct(service $cache, config $config, driver_interface $db, user $user, string $search_results_table) + { + $this->cache = $cache; + $this->config = $config; + $this->db = $db; + $this->user = $user; + $this->search_results_table = $search_results_table; + } + + /** + * Retrieves cached search results + * + * @param string $search_key an md5 string generated from all the passed search options to identify the + * results + * @param int & $result_count will contain the number of all results for the search (not only for the current + * page) + * @param array &$id_ary is filled with the ids belonging to the requested page that are stored in the + * cache + * @param int & $start indicates the first index of the page + * @param int $per_page number of ids each page is supposed to contain + * @param string $sort_dir is either a or d representing ASC and DESC + * + * @return int self::SEARCH_RESULT_NOT_IN_CACHE or self::SEARCH_RESULT_IN_CACHE or self::SEARCH_RESULT_INCOMPLETE + */ + protected function obtain_ids(string $search_key, int &$result_count, array &$id_ary, int &$start, int $per_page, string $sort_dir): int + { + if (!($stored_ids = $this->cache->get('_wiki_search_results_' . $search_key))) + { + // no search results cached for this search_key + return self::WIKI_SEARCH_RESULT_NOT_IN_CACHE; + } + else + { + $result_count = $stored_ids[-1]; + $reverse_ids = $stored_ids[-2] != $sort_dir; + $complete = true; + + // Change start parameter in case out of bounds + if ($result_count) + { + if ($start < 0) + { + $start = 0; + } + else if ($start >= $result_count) + { + $start = floor(($result_count - 1) / $per_page) * $per_page; + } + } + + // change the start to the actual end of the current request if the sort direction differs + // from the direction in the cache and reverse the ids later + if ($reverse_ids) + { + $start = $result_count - $start - $per_page; + + // the user requested a page past the last index + if ($start < 0) + { + return self::WIKI_SEARCH_RESULT_NOT_IN_CACHE; + } + } + + for ($i = $start, $n = $start + $per_page; ($i < $n) && ($i < $result_count); $i++) + { + if (!isset($stored_ids[$i])) + { + $complete = false; + } + else + { + $id_ary[] = $stored_ids[$i]; + } + } + unset($stored_ids); + + if ($reverse_ids) + { + $id_ary = array_reverse($id_ary); + } + + if (!$complete) + { + return self::WIKI_SEARCH_RESULT_INCOMPLETE; + } + return self::WIKI_SEARCH_RESULT_IN_CACHE; + } + } + + /** + * Caches post/topic ids + * + * @param string $search_key an md5 string generated from all the passed search options to identify the results + * @param string $keywords contains the keywords as entered by the user + * @param array $author_ary an array of author ids, if the author should be ignored during the search the + * array is empty + * @param int $result_count contains the number of all results for the search (not only for the current page) + * @param array $id_ary contains a list of post or topic ids that shall be cached, the first element + * must have the absolute index $start in the result set. + * @param int $start indicates the first index of the page + * @param string $sort_dir is either a or d representing ASC and DESC + * + * @return void + */ + protected function save_ids(string $search_key, string $keywords, array $author_ary, int $result_count, array &$id_ary, int $start, string $sort_dir): void + { + $length = min(count($id_ary), $this->config['search_block_size']); + + // nothing to cache so exit + if (!$length) + { + return; + } + + $store_ids = array_slice($id_ary, 0, $length); + + // create a new result set if there is none for this search_key yet + // or add the ids to the existing result set + if (!($store = $this->cache->get('_wiki_search_results_' . $search_key))) + { + // add the current keywords to the recent searches in the cache which are listed on the search page + if (!empty($keywords) || count($author_ary)) + { + $sql = 'SELECT search_time + FROM ' . $this->search_results_table . ' + WHERE search_key = \'' . $this->db->sql_escape($search_key) . '\''; + $result = $this->db->sql_query($sql); + + if (!$this->db->sql_fetchrow($result)) + { + $sql_ary = [ + 'search_key' => $search_key, + 'search_time' => time(), + 'search_keywords' => $keywords, + 'search_authors' => ' ' . implode(' ', $author_ary) . ' ', + ]; + + $sql = 'INSERT INTO ' . $this->search_results_table . ' ' . $this->db->sql_build_array('INSERT', $sql_ary); + $this->db->sql_query($sql); + } + $this->db->sql_freeresult($result); + } + + $sql = 'UPDATE ' . USERS_TABLE . ' + SET user_last_search = ' . time() . ' + WHERE user_id = ' . (int) $this->user->data['user_id']; + $this->db->sql_query($sql); + + $store = [-1 => $result_count, -2 => $sort_dir]; + $id_range = range($start, $start + $length - 1); + } + else + { + // we use one set of results for both sort directions so we have to calculate the sizes + // for the reversed array and we also have to reverse the ids themselves + if ($store[-2] != $sort_dir) + { + $store_ids = array_reverse($store_ids); + $id_range = range($store[-1] - $start - $length, $store[-1] - $start - 1); + } + else + { + $id_range = range($start, $start + $length - 1); + } + } + + $store_ids = array_combine($id_range, $store_ids); + + // append the ids + if (is_array($store_ids)) + { + $store += $store_ids; + + // if the cache is too big + if (count($store) - 2 > 20 * $this->config['search_block_size']) + { + // remove everything in front of two blocks in front of the current start index + for ($i = 0, $n = $id_range[0] - 2 * $this->config['search_block_size']; $i < $n; $i++) + { + if (isset($store[$i])) + { + unset($store[$i]); + } + } + + // remove everything after two blocks after the current stop index + end($id_range); + for ($i = $store[-1] - 1, $n = current($id_range) + 2 * $this->config['search_block_size']; $i > $n; $i--) + { + if (isset($store[$i])) + { + unset($store[$i]); + } + } + } + $this->cache->put('_wiki_search_results_' . $search_key, $store, $this->config['search_store_results']); + + $sql = 'UPDATE ' . $this->search_results_table . ' + SET search_time = ' . time() . ' + WHERE search_key = \'' . $this->db->sql_escape($search_key) . '\''; + $this->db->sql_query($sql); + } + + unset($store, $store_ids, $id_range); + } + + /** + * Removes old entries from the search results table and removes searches with keywords that contain a word in + * $words. + */ + protected function destroy_cache(array $words, $authors = false): void + { + // clear all searches that searched for the specified words + if (count($words)) + { + $sql_where = ''; + foreach ($words as $word) + { + $sql_where .= ' OR search_keywords ' . $this->db->sql_like_expression($this->db->get_any_char() . $word . $this->db->get_any_char()); + } + + $sql = 'SELECT search_key + FROM ' . $this->search_results_table . " + WHERE search_keywords LIKE '%*%' " . $sql_where; + $result = $this->db->sql_query($sql); + + while ($row = $this->db->sql_fetchrow($result)) + { + $this->cache->destroy('_wiki_search_results_' . $row['search_key']); + } + $this->db->sql_freeresult($result); + } + + // clear all searches that searched for the specified authors + if (is_array($authors) && count($authors)) + { + $sql_where = ''; + foreach ($authors as $author) + { + $sql_where .= (($sql_where) ? ' OR ' : '') . 'search_authors ' . $this->db->sql_like_expression($this->db->get_any_char() . ' ' . (int) $author . ' ' . $this->db->get_any_char()); + } + + $sql = 'SELECT search_key FROM ' . $this->search_results_table . ' WHERE ' . $sql_where; + $result = $this->db->sql_query($sql); + + while ($row = $this->db->sql_fetchrow($result)) + { + $this->cache->destroy('_wiki_search_results_' . $row['search_key']); + } + $this->db->sql_freeresult($result); + } + + $sql = 'DELETE + FROM ' . $this->search_results_table . ' + WHERE search_time < ' . (time() - (int) $this->config['search_store_results']); + $this->db->sql_query($sql); + } + + /** + * @param int $article_counter + * @return array|null + */ + public function create_index(int &$article_counter = 0): array|null + { + $max_article_id = $this->get_max_article_id(); + + $starttime = microtime(true); + $row_count = 0; + + while (still_on_time() && $article_counter <= $max_article_id) + { + $rows = $this->get_articles_batch_after($article_counter); + + if ($this->db->sql_buffer_nested_transactions()) + { + $rows = iterator_to_array($rows); + } + + foreach ($rows as $row) + { + $this->index('add', (int) $row['article_id'], $row['article_body'], $row['article_title'], $row['article_description'], (int) $row['author_id']); + $row_count++; + $article_counter = $row['article_id']; + } + } + + // pretend the number of posts was as big as the number of ids we indexed so far + // just an estimation as it includes deleted posts + $num_articles = $this->config['wiki_num_articles']; + $this->config['wiki_num_articles'] = min($this->config['wiki_num_articles'], $article_counter); + $this->tidy(); + $this->config['wiki_num_articles'] = $num_articles; + + if ($article_counter < $max_article_id) + { + $totaltime = microtime(true) - $starttime; + $rows_per_second = $row_count / $totaltime; + + return [ + 'row_count' => $row_count, + 'post_counter' => $article_counter, + 'max_post_id' => $max_article_id, + 'rows_per_second' => $rows_per_second, + ]; + } + + return null; + } + + /** + * @param int|null $article_counter + * @return array|null + */ + public function delete_index(int|null &$article_counter = null): array|null + { + $max_article_id = $this->get_max_article_id(); + + $starttime = microtime(true); + $row_count = 0; + while (still_on_time() && $article_counter <= $max_article_id) + { + $rows = $this->get_articles_batch_after($article_counter); + $ids = $authors = $category_ids = []; + foreach ($rows as $row) + { + $ids[] = $row['article_id']; + $authors[] = $row['author_id']; + $category_ids[] = $row['article_category_id']; + } + $row_count += count($ids); + + if (count($ids)) + { + $this->index_remove($ids, $authors); + $article_counter = $ids[count($ids) - 1]; + } + } + + if ($article_counter < $max_article_id) + { + $totaltime = microtime(true) - $starttime; + $rows_per_second = $row_count / $totaltime; + + return [ + 'row_count' => $row_count, + 'post_counter' => $article_counter, + 'max_post_id' => $max_article_id, + 'rows_per_second' => $rows_per_second, + ]; + } + + return null; + } + + /** + * Get batch of posts after id + * + * @param int $article_id + * @return \Generator + */ + protected function get_articles_batch_after(int $article_id): \Generator + { + $sql = 'SELECT article_id, author_id, article_category_id, article_body, article_title, article_description + FROM ' . $this->articles_table . ' + WHERE article_id > ' . $article_id . ' + ORDER BY article_id ASC'; + $result = $this->db->sql_query_limit($sql, self::BATCH_SIZE); + + while ($row = $this->db->sql_fetchrow($result)) + { + yield $row; + } + + $this->db->sql_freeresult($result); + } + + /** + * Get post with higher id + */ + protected function get_max_article_id(): int + { + $sql = 'SELECT MAX(article_id) as max_article_id + FROM ' . $this->articles_table; + $result = $this->db->sql_query($sql); + $max_article_id = (int) $this->db->sql_fetchfield('max_article_id'); + $this->db->sql_freeresult($result); + + return $max_article_id; + } + + /** + * @return string + */ + public function get_type(): string + { + return static::class; + } +} diff --git a/search/backend/wiki_fulltext_mysql.php b/search/backend/wiki_fulltext_mysql.php new file mode 100644 index 0000000..2480ae4 --- /dev/null +++ b/search/backend/wiki_fulltext_mysql.php @@ -0,0 +1,884 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbbmodders\wiki\search\backend; + +use phpbb\config\config; +use phpbb\db\driver\driver_interface; +use phpbb\language\language; +use phpbb\user; +use RuntimeException; + +class wiki_fulltext_mysql extends wiki_base implements wiki_search_backend_interface +{ + /** + * Associative array holding index stats + * + * @var array + */ + protected $stats = []; + + /** + * Holds the words entered by user, obtained by splitting the entered query on whitespace + * + * @var array + */ + protected $split_words = []; + + /** + * @var language + */ + protected $language; + + /** + * Associative array stores the min and max word length to be searched + * + * @var array + */ + protected $word_length = []; + + /** + * Contains tidied search query. + * Operators are prefixed in search query and common words excluded + * + * @var string + */ + protected $search_query = ''; + + /** + * Contains common words. + * Common words are words with length less/more than min/max length + * + * @var array + */ + protected $common_words = []; + + /** @var string */ + protected $articles_table; + + /** @var string */ + protected string $search_results_table; + + /** + * Constructor + * Creates a new \phpbb\search\backend\fulltext_mysql, which is used as a search backend + * + * @param config $config Config object + * @param driver_interface $db Database object + * @param language $language + * @param user $user User object + * @param string $articles_table Articles_table + * @param string $search_results_table Search_results_table + * @param string $phpbb_root_path Relative path to phpBB root + * @param string $phpEx PHP file extension + */ + public function __construct( + config $config, + driver_interface $db, + language $language, + user $user, + string $articles_table, + string $search_results_table, + string $phpbb_root_path, + string $phpEx + ) + { + global $cache; + + parent::__construct($cache, $config, $db, $user, $search_results_table); + $this->language = $language; + + $this->word_length = ['min' => $this->config['fulltext_mysql_min_word_len'], 'max' => $this->config['fulltext_mysql_max_word_len']]; + + $this->articles_table = $articles_table; + $this->search_results_table = $search_results_table; + + /** + * Load the UTF tools + */ + if (!function_exists('utf8_strlen')) + { + include($phpbb_root_path . 'includes/utf/utf_tools.' . $phpEx); + } + } + + + /** + * {@inheritdoc} + */ + public function get_name(): string + { + return 'Wiki MySQL Fulltext'; + } + + /** + * {@inheritdoc} + */ + public function is_available(): bool + { + // Check if we are using mysql + if ($this->db->get_sql_layer() != 'mysqli') + { + return false; + } + + return true; + } + + /** + * {@inheritdoc} + */ + public function init() + { + if (!$this->is_available()) + { + return $this->language->lang('FULLTEXT_MYSQL_INCOMPATIBLE_DATABASE'); + } + + $result = $this->db->sql_query('SHOW TABLE STATUS LIKE \'' . $this->articles_table . '\''); + $info = $this->db->sql_fetchrow($result); + $this->db->sql_freeresult($result); + + $engine = $info['Engine'] ?? $info['Type'] ?? ''; + + $fulltext_supported = $engine === 'Aria' || $engine === 'MyISAM' + /** + * FULLTEXT is supported on InnoDB since MySQL 5.6.4 according to + * http://dev.mysql.com/doc/refman/5.6/en/innodb-storage-engine.html + * We also require https://bugs.mysql.com/bug.php?id=67004 to be + * fixed for proper overall operation. Hence we require 5.6.8. + */ + || ($engine === 'InnoDB' + && phpbb_version_compare($this->db->sql_server_info(true), '5.6.8', '>=')); + + if (!$fulltext_supported) + { + return $this->language->lang('FULLTEXT_MYSQL_NOT_SUPPORTED'); + } + + $sql = 'SHOW VARIABLES + LIKE \'%ft\_%\''; + $result = $this->db->sql_query($sql); + + $mysql_info = []; + while ($row = $this->db->sql_fetchrow($result)) + { + $mysql_info[$row['Variable_name']] = $row['Value']; + } + $this->db->sql_freeresult($result); + + if ($engine === 'MyISAM') + { + $this->config->set('fulltext_mysql_max_word_len', $mysql_info['ft_max_word_len']); + $this->config->set('fulltext_mysql_min_word_len', $mysql_info['ft_min_word_len']); + } + else if ($engine === 'InnoDB') + { + $this->config->set('fulltext_mysql_max_word_len', $mysql_info['innodb_ft_max_token_size']); + $this->config->set('fulltext_mysql_min_word_len', $mysql_info['innodb_ft_min_token_size']); + } + + return false; + } + + /** + * {@inheritdoc} + */ + public function get_search_query(): string + { + return $this->search_query; + } + + /** + * {@inheritdoc} + */ + public function get_common_words(): array + { + return $this->common_words; + } + + /** + * {@inheritdoc} + */ + public function get_word_length() + { + return $this->word_length; + } + + /** + * {@inheritdoc} + */ + public function split_keywords(string &$keywords, string $terms): bool + { + if ($terms == 'all') + { + $match = ['#\sand\s#iu', '#\sor\s#iu', '#\snot\s#iu', '#(^|\s)\+#', '#(^|\s)-#', '#(^|\s)\|#']; + $replace = [' +', ' |', ' -', ' +', ' -', ' |']; + + $keywords = preg_replace($match, $replace, $keywords); + } + + // Filter out as above + $split_keywords = preg_replace("#[\n\r\t]+#", ' ', trim(htmlspecialchars_decode($keywords, ENT_COMPAT))); + + // Split words + $split_keywords = preg_replace('#([^\p{L}\p{N}\'*"()])#u', '$1$1', str_replace('\'\'', '\' \'', trim($split_keywords))); + $matches = []; + preg_match_all('#(?:[^\p{L}\p{N}*"()]|^)([+\-|]?(?:[\p{L}\p{N}*"()]+\'?)*[\p{L}\p{N}*"()])(?:[^\p{L}\p{N}*"()]|$)#u', $split_keywords, $matches); + $this->split_words = $matches[1]; + + // We limit the number of allowed keywords to minimize load on the database + if ($this->config['max_num_search_keywords'] && count($this->split_words) > $this->config['max_num_search_keywords']) + { + trigger_error($this->language->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], count($this->split_words))); + } + + // to allow phrase search, we need to concatenate quoted words + $tmp_split_words = []; + $phrase = ''; + foreach ($this->split_words as $word) + { + if ($phrase) + { + $phrase .= ' ' . $word; + if (strpos($word, '"') !== false && substr_count($word, '"') % 2 === 1) + { + $tmp_split_words[] = $phrase; + $phrase = ''; + } + } + else if (strpos($word, '"') !== false && substr_count($word, '"') % 2 === 1) + { + $phrase = $word; + } + else + { + $tmp_split_words[] = $word; + } + } + if ($phrase) + { + $tmp_split_words[] = $phrase; + } + + $this->split_words = $tmp_split_words; + + unset($tmp_split_words); + unset($phrase); + + foreach ($this->split_words as $i => $word) + { + // Check for not allowed search queries for InnoDB. + // We assume similar restrictions for MyISAM, which is usually even + // slower but not as restrictive as InnoDB. + // InnoDB full-text search does not support the use of a leading + // plus sign with wildcard ('+*'), a plus and minus sign + // combination ('+-'), or leading a plus and minus sign combination. + // InnoDB full-text search only supports leading plus or minus signs. + // For example, InnoDB supports '+apple' but does not support 'apple+'. + // Specifying a trailing plus or minus sign causes InnoDB to report + // a syntax error. InnoDB full-text search does not support the use + // of multiple operators on a single search word, as in this example: + // '++apple'. Use of multiple operators on a single search word + // returns a syntax error to standard out. + // Also, ensure that the wildcard character is only used at the + // end of the line as it's intended by MySQL. + if (preg_match('#^(\+[+-]|\+\*|.+[+-]$|.+\*(?!$))#', $word)) + { + unset($this->split_words[$i]); + continue; + } + + $clean_word = preg_replace('#^[+\-|"]#', '', $word); + + // check word length + $clean_len = utf8_strlen(str_replace('*', '', $clean_word)); + if (($clean_len < $this->config['fulltext_mysql_min_word_len']) || ($clean_len > $this->config['fulltext_mysql_max_word_len'])) + { + $this->common_words[] = $word; + unset($this->split_words[$i]); + } + } + + $this->search_query = ''; + if ($terms == 'any') + { + foreach ($this->split_words as $word) + { + if ((strpos($word, '+') === 0) || (strpos($word, '-') === 0) || (strpos($word, '|') === 0)) + { + $word = substr($word, 1); + } + $this->search_query .= $word . ' '; + } + } + else + { + foreach ($this->split_words as $word) + { + if ((strpos($word, '+') === 0) || (strpos($word, '-') === 0)) + { + $this->search_query .= $word . ' '; + } + else if (strpos($word, '|') === 0) + { + $this->search_query .= substr($word, 1) . ' '; + } + else + { + $this->search_query .= '+' . $word . ' '; + } + } + } + + $this->search_query = utf8_htmlspecialchars($this->search_query); + + if ($this->search_query) + { + $this->split_words = array_values($this->split_words); + sort($this->split_words); + return true; + } + return false; + } + + /** + * {@inheritdoc} + */ + public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No keywords? No posts + if (!$this->search_query) + { + return false; + } + + // generate a search_key from all the options to identify the results + $search_key_array = [ + implode(', ', $this->split_words), + $type, + $fields, + $terms, + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + '', + implode(',', $author_ary), + ]; + $search_key = md5(implode('#', $search_key_array)); + + if ($start < 0) + { + $start = 0; + } + + // try reading the results from cache + $result_count = 0; + + if ($this->obtain_ids($search_key, $result_count, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $result_count; + } + + $id_ary = []; + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + // Build some display specific sql strings + switch ($fields) + { + case 'titleonly': + $sql_match = 'p.article_title'; + break; + + case 'descronly': + $sql_match = 'p.article_description'; + break; + + case 'msgonly': + $sql_match = 'p.article_body'; + break; + + default: + $sql_match = 'p.article_title, p.article_body, p.article_description'; + break; + } + + $search_query = $this->search_query; + + $sql_select = 'p.article_id'; + $field = 'article_id'; + + if (count($author_ary) && $author_name) + { + // first one matches post of registered users, second one guests and deleted users + $sql_author = ' AND (' . $this->db->sql_in_set('p.author_id', array_diff($author_ary, [ANONYMOUS]), false, true) . " OR p.author = '" . $this->db->sql_escape($author_name) . "')"; + } + else if (count($author_ary)) + { + $sql_author = ' AND ' . $this->db->sql_in_set('p.author_id', $author_ary); + } + else + { + $sql_author = ''; + } + + $sql_where_options = ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true) : ''; + $sql_where_options .= $sql_author; + $sql_where_options .= ($sort_days) ? ' AND p.article_date >= ' . (time() - ($sort_days * 86400)) : ''; + $sql_where_options .= ' AND p.approved = 1 '; + + $sql = 'SELECT ' . $sql_select . ' FROM ' . $this->articles_table . ' p + WHERE MATCH (' . $sql_match . ") AGAINST ('" . $this->db->sql_escape(htmlspecialchars_decode($this->search_query, ENT_COMPAT)) . "' IN BOOLEAN MODE)" . ' + ' . $sql_where_options . ' + ORDER BY ' . $sql_sort; + $this->db->sql_return_on_error(true); + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + + // if the total result count is not cached yet, retrieve it from the db + if (!$result_count) + { + $sql_found_rows = str_replace('SELECT ' . $sql_select, 'SELECT COUNT(*) as result_count', $sql); + $result = $this->db->sql_query($sql_found_rows); + $result_count = (int) $this->db->sql_fetchfield('result_count'); + $this->db->sql_freeresult($result); + + if (!$result_count) + { + return false; + } + } + + if ($start >= $result_count) + { + $start = floor(($result_count - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + } + + // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page + $this->save_ids($search_key, implode(' ', $this->split_words), $author_ary, $result_count, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $result_count; + } + + /** + * {@inheritdoc} + */ + public function author_search(string $type, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No author? No posts + if (!count($author_ary)) + { + return 0; + } + + // generate a search_key from all the options to identify the results + $search_key_array = [ + '', + $type, + '', + '', + '', + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + '', + implode(',', $author_ary), + $author_name, + ]; + $search_key = md5(implode('#', $search_key_array)); + + if ($start < 0) + { + $start = 0; + } + + // try reading the results from cache + $result_count = 0; + if ($this->obtain_ids($search_key, $result_count, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $result_count; + } + + $id_ary = []; + + // Create some display specific sql strings + $sql_author = $this->db->sql_in_set('p.author_id', $author_ary) . ' AND p.approved=1 '; + + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true) : ''; + $sql_category_id = ''; + $sql_time = ($sort_days) ? ' AND p.article_date >= ' . (time() - ($sort_days * 86400)) : ''; + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + // If the cache was completely empty count the results + $sql_select = 'p.article_id'; +// $sql_select .= $sort_by_sql[$sort_key] ? ", {$sort_by_sql[$sort_key]}" : ''; + + // Build the query for really selecting the post_ids + $sql = 'SELECT ' . $sql_select . ' FROM ' . $this->articles_table . ' p' . " + WHERE $sql_author + $sql_category_id + $sql_fora + $sql_time + ORDER BY $sql_sort"; + $field = 'article_id'; + + // Only read one block of posts from the db and then cache it + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + // retrieve the total result count if needed + if (!$result_count) + { + $sql_found_rows = str_replace("SELECT $sql_select", 'SELECT COUNT(*) as result_count', $sql); + $result = $this->db->sql_query($sql_found_rows); + $result_count = (int) $this->db->sql_fetchfield('result_count'); + + $this->db->sql_freeresult($result); + + if (!$result_count) + { + return false; + } + } + + if ($start >= $result_count) + { + $start = floor(($result_count - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + } + + if (count($id_ary)) + { + $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $result_count; + } + return false; + } + + /** + * {@inheritdoc} + */ + public function supports_phrase_search(): bool + { + return false; + } + + /** + * {@inheritdoc} + */ + public function index(string $mode, int $article_id, string &$message, string &$subject, string &$description, int $poster_id) + { + // Split old and new post/subject to obtain array of words + $split_text = $this->split_message($message); + $split_title = ($subject) ? $this->split_message($subject) : []; + $split_descr = ($description) ? $this->split_message($description) : []; + + $words = array_unique(array_merge($split_text, $split_title, $split_descr)); + + unset($split_text, $split_title, $split_descr); + + // destroy cached search results containing any of the words removed or added + $this->destroy_cache($words, [$poster_id]); + + unset($words); + } + + /** + * {@inheritdoc} + */ + public function index_remove(array $article_ids, array $author_ids): void + { + $this->destroy_cache([], array_unique($author_ids)); + } + + /** + * Destroy old cache entries + */ + public function tidy(): void + { + // destroy too old cached search results + $this->destroy_cache([]); + + $this->config->set('search_last_gc', time(), false); + } + + /** + * {@inheritdoc} + */ + public function create_index(int &$post_counter = 0): array|null + { + // Make sure we can actually use MySQL with fulltext indexes + if ($error = $this->init()) + { + throw new RuntimeException($error); + } + + if (empty($this->stats)) + { + $this->get_stats(); + } + + $alter_list = []; + + if (!isset($this->stats['article_title'])) + { + $alter_entry = []; + $alter_entry[] = 'MODIFY article_title varchar(255) COLLATE utf8_unicode_ci DEFAULT \'\' NOT NULL'; + $alter_entry[] = 'ADD FULLTEXT (article_title)'; + $alter_list[] = $alter_entry; + } + + if (!isset($this->stats['article_description'])) + { + $alter_entry = []; + $alter_entry[] = 'MODIFY article_description mediumtext COLLATE utf8_unicode_ci NOT NULL'; + $alter_entry[] = 'ADD FULLTEXT (article_description)'; + $alter_list[] = $alter_entry; + } + + if (!isset($this->stats['article_body'])) + { + $alter_entry = []; + $alter_entry[] = 'MODIFY article_body mediumtext COLLATE utf8_unicode_ci NOT NULL'; + $alter_entry[] = 'ADD FULLTEXT article_body (article_body)'; + $alter_list[] = $alter_entry; + } + + if (!isset($this->stats['article_content'])) + { + $alter_entry = []; + $alter_entry[] = 'MODIFY article_body mediumtext COLLATE utf8_unicode_ci NOT NULL'; + $alter_entry[] = 'ADD FULLTEXT article_content (article_body, article_title, article_description)'; + $alter_list[] = $alter_entry; + } + + $sql_queries = []; + + foreach ($alter_list as $alter) + { + $sql_queries[] = 'ALTER TABLE ' . $this->articles_table . ' ' . implode(', ', $alter); + } + + $stats = $this->stats; + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } + + $this->stats = []; + $this->db->sql_query('TRUNCATE TABLE ' . $this->search_results_table); + + return null; + } + + /** + * {@inheritdoc} + */ + public function delete_index(int|null &$post_counter = null): array|null + { + // Make sure we can actually use MySQL with fulltext indexes + if ($error = $this->init()) + { + throw new RuntimeException($error); + } + + if (empty($this->stats)) + { + $this->get_stats(); + } + + $alter = []; + + if (isset($this->stats['article_title'])) + { + $alter[] = 'DROP INDEX article_title'; + } + + if (isset($this->stats['article_description'])) + { + $alter[] = 'DROP INDEX article_description'; + } + + if (isset($this->stats['article_body'])) + { + $alter[] = 'DROP INDEX article_body'; + } + + if (isset($this->stats['article_content'])) + { + $alter[] = 'DROP INDEX article_content'; + } + + $sql_queries = []; + if (count($alter)) + { + $sql_queries[] = 'ALTER TABLE ' . $this->articles_table . ' ' . implode(', ', $alter); + } + + $stats = $this->stats; + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } + + $this->stats = []; + $this->db->sql_query('TRUNCATE TABLE ' . $this->search_results_table); + + return null; + } + + /** + * {@inheritdoc} + */ + public function index_created(): bool + { + if (empty($this->stats)) + { + $this->get_stats(); + } + + return isset($this->stats['article_title']) && isset($this->stats['article_content']) && isset($this->stats['article_description']) && isset($this->stats['article_body']); + } + + /** + * {@inheritdoc} + */ + public function index_stats() + { + if (empty($this->stats)) + { + $this->get_stats(); + } + + return [ + $this->language->lang('FULLTEXT_MYSQL_TOTAL_POSTS') => ($this->index_created()) ? $this->stats['total_posts'] : 0, + ]; + } + + /** + * Computes the stats and store them in the $this->stats associative array + */ + protected function get_stats() + { + if (strpos($this->db->get_sql_layer(), 'mysql') === false) + { + $this->stats = []; + return; + } + + $sql = 'SHOW INDEX + FROM ' . $this->articles_table; + $result = $this->db->sql_query($sql); + + while ($row = $this->db->sql_fetchrow($result)) + { + // deal with older MySQL versions which didn't use Index_type + $index_type = (isset($row['Index_type'])) ? $row['Index_type'] : $row['Comment']; + + if ($index_type == 'FULLTEXT') + { + if ($row['Key_name'] == 'article_title') + { + $this->stats['article_title'] = $row; + } + else if ($row['Key_name'] == 'article_description') + { + $this->stats['article_description'] = $row; + } + + else if ($row['Key_name'] == 'article_body') + { + $this->stats['article_body'] = $row; + } + else if ($row['Key_name'] == 'article_content') + { + $this->stats['article_content'] = $row; + } + } + } + $this->db->sql_freeresult($result); + + $this->stats['total_posts'] = empty($this->stats) ? 0 : $this->db->get_estimated_row_count($this->articles_table); + } + + /** + * Turns text into an array of words + * + * @param string $text contains post text/subject + * + * @return array + */ + protected function split_message($text): array + { + // Split words + $text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text))); + $matches = []; + preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches); + $text = $matches[1]; + + // remove too short or too long words + $text = array_values($text); + for ($i = 0, $n = count($text); $i < $n; $i++) + { + $text[$i] = trim($text[$i]); + if (utf8_strlen($text[$i]) < $this->config['fulltext_mysql_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_mysql_max_word_len']) + { + unset($text[$i]); + } + } + + return array_values($text); + } +} diff --git a/search/backend/wiki_fulltext_native.php b/search/backend/wiki_fulltext_native.php new file mode 100644 index 0000000..96d0d9e --- /dev/null +++ b/search/backend/wiki_fulltext_native.php @@ -0,0 +1,1684 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbbmodders\wiki\search\backend; + +use phpbb\config\config; +use phpbb\db\driver\driver_interface; +use phpbb\language\language; +use phpbb\user; + +/** + * phpBB's own db driven fulltext search, version 2 + */ +class wiki_fulltext_native extends wiki_base implements wiki_search_backend_interface +{ + protected const UTF8_HANGUL_FIRST = "\xEA\xB0\x80"; + protected const UTF8_HANGUL_LAST = "\xED\x9E\xA3"; + protected const UTF8_CJK_FIRST = "\xE4\xB8\x80"; + protected const UTF8_CJK_LAST = "\xE9\xBE\xBB"; + protected const UTF8_CJK_B_FIRST = "\xF0\xA0\x80\x80"; + protected const UTF8_CJK_B_LAST = "\xF0\xAA\x9B\x96"; + + /** + * Associative array holding index stats + * + * @var array + */ + protected $stats = []; + + /** + * Associative array stores the min and max word length to be searched + * + * @var array + */ + protected $word_length = []; + + /** + * Contains tidied search query. + * Operators are prefixed in search query and common words excluded + * + * @var string + */ + protected $search_query = ''; + + /** + * Contains common words. + * Common words are words with length less/more than min/max length + * + * @var array + */ + protected $common_words = []; + + /** + * Post ids of posts containing words that are to be included + * + * @var array + */ + protected $must_contain_ids = []; + + /** + * Post ids of posts containing words that should not be included + * + * @var array + */ + protected $must_not_contain_ids = []; + + /** + * Post ids of posts containing at least one word that needs to be excluded + * + * @var array + */ + protected $must_exclude_one_ids = []; + + /** + * Relative path to board root + * + * @var string + */ + protected $phpbb_root_path; + + /** + * PHP Extension + * + * @var string + */ + protected $php_ext; + + /** @var language */ + protected $language; + + /** @var string */ + protected $articles_table; + + /** @var string */ + protected string $search_results_table; + + /** @var string */ + protected $wordmatch_table; + + /** @var string */ + protected $wordlist_table; + + /** + * Initialises the fulltext_native search backend with min/max word length + * + * @param config $config Config object + * @param driver_interface $db Database object + * @param language $language + * @param user $user User object + * @param string $articles_table Articles_table + * @param string $search_results_table Search_results_table + * @param string $wordmatch_table Wordmatch_table + * @param string $wordlist_table Wordlist_table + * @param string $phpbb_root_path phpBB root path + * @param string $phpEx PHP file extension + */ + public function __construct( + config $config, + driver_interface $db, + language $language, + user $user, + string $articles_table, + string $search_results_table, + string $wordmatch_table, + string $wordlist_table, + string $phpbb_root_path, + string $phpEx + ) + { + global $cache; + + parent::__construct($cache, $config, $db, $user, $search_results_table); + $this->language = $language; + + $this->articles_table = $articles_table; + $this->wordmatch_table = $wordmatch_table; + $this->wordlist_table = $wordlist_table; + + $this->phpbb_root_path = $phpbb_root_path; + $this->php_ext = $phpEx; + + $this->word_length = ['min' => (int) $this->config['fulltext_native_min_chars'], 'max' => (int) $this->config['fulltext_native_max_chars']]; + + /** + * Load the UTF tools + */ + if (!function_exists('utf8_decode_ncr')) + { + include($this->phpbb_root_path . 'includes/utf/utf_tools.' . $this->php_ext); + } + } + + /** + * {@inheritdoc} + */ + public function get_name(): string + { + return 'Wiki Native Fulltext'; + } + + /** + * {@inheritdoc} + */ + public function is_available(): bool + { + return true; + } + + /** + * {@inheritdoc} + */ + public function init() + { + return false; + } + + /** + * {@inheritdoc} + */ + public function get_search_query(): string + { + return $this->search_query; + } + + /** + * {@inheritdoc} + */ + public function get_common_words(): array + { + return $this->common_words; + } + + /** + * {@inheritdoc} + */ + public function get_word_length() + { + return $this->word_length; + } + + /** + * {@inheritdoc} + */ + public function split_keywords(string &$keywords, string $terms): bool + { + $tokens = '+-|()* '; + + $keywords = trim($this->cleanup($keywords, $tokens)); + + // allow word|word|word without brackets + if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false)) + { + $keywords = '(' . $keywords . ')'; + } + + $open_bracket = $space = false; + for ($i = 0, $n = strlen($keywords); $i < $n; $i++) + { + if ($open_bracket !== false) + { + switch ($keywords[$i]) + { + case ')': + if ($open_bracket + 1 == $i) + { + $keywords[$i - 1] = '|'; + $keywords[$i] = '|'; + } + $open_bracket = false; + break; + case '(': + $keywords[$i] = '|'; + break; + case '+': + case '-': + case ' ': + $keywords[$i] = '|'; + break; + case '*': + // $i can never be 0 here since $open_bracket is initialised to false + if (strpos($tokens, $keywords[$i - 1]) !== false && ($i + 1 === $n || strpos($tokens, $keywords[$i + 1]) !== false)) + { + $keywords[$i] = '|'; + } + break; + } + } + else + { + switch ($keywords[$i]) + { + case ')': + $keywords[$i] = ' '; + break; + case '(': + $open_bracket = $i; + $space = false; + break; + case '|': + $keywords[$i] = ' '; + break; + case '-': + // Ignore hyphen if followed by a space + if (isset($keywords[$i + 1]) && $keywords[$i + 1] == ' ') + { + $keywords[$i] = ' '; + } + else + { + $space = $keywords[$i]; + } + break; + case '+': + $space = $keywords[$i]; + break; + case ' ': + if ($space !== false) + { + $keywords[$i] = $space; + } + break; + default: + $space = false; + } + } + } + + if ($open_bracket !== false) + { + $keywords .= ')'; + } + + $match = [ + '# +#', + '#\|\|+#', + '#(\+|\-)(?:\+|\-)+#', + '#\(\|#', + '#\|\)#', + ]; + $replace = [ + ' ', + '|', + '$1', + '(', + ')', + ]; + + $keywords = preg_replace($match, $replace, $keywords); + $num_keywords = count(explode(' ', $keywords)); + + // We limit the number of allowed keywords to minimize load on the database + if ($this->config['max_num_search_keywords'] && $num_keywords > $this->config['max_num_search_keywords']) + { + trigger_error($this->language->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', (int) $this->config['max_num_search_keywords'], $num_keywords)); + } + + // $keywords input format: each word separated by a space, words in a bracket are not separated + + // the user wants to search for any word, convert the search query + if ($terms == 'any') + { + $words = []; + + preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words); + if (count($words[1])) + { + $keywords = '(' . implode('|', $words[1]) . ')'; + } + } + + // Remove non trailing wildcards from each word to prevent a full table scan (it's now using the database index) + $match = '#\*(?!$|\s)#'; + $replace = '$1'; + $keywords = preg_replace($match, $replace, $keywords); + + // Only allow one wildcard in the search query to limit the database load + $match = '#\*#'; + $count_wildcards = substr_count($keywords, '*'); + + // Reverse the string to remove all wildcards except the first one + $keywords = strrev(preg_replace($match, $replace, strrev($keywords), $count_wildcards - 1)); + unset($count_wildcards); + + // set the search_query which is shown to the user + $this->search_query = $keywords; + + $exact_words = []; + preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words); + $exact_words = $exact_words[1]; + + $common_ids = $words = []; + + if (count($exact_words)) + { + $sql = 'SELECT word_id, word_text, word_common + FROM ' . $this->wordlist_table . ' + WHERE ' . $this->db->sql_in_set('word_text', $exact_words) . ' + ORDER BY word_count ASC'; + $result = $this->db->sql_query($sql); + + // store an array of words and ids, remove common words + while ($row = $this->db->sql_fetchrow($result)) + { + if ($row['word_common']) + { + $this->common_words[] = $row['word_text']; + $common_ids[$row['word_text']] = (int) $row['word_id']; + continue; + } + + $words[$row['word_text']] = (int) $row['word_id']; + } + $this->db->sql_freeresult($result); + } + + // Handle +, - without preceding whitespace character + $match = ['#(\S)\+#', '#(\S)-#']; + $replace = ['$1 +', '$1 +']; + + $keywords = preg_replace($match, $replace, $keywords); + + // now analyse the search query, first split it using the spaces + $query = explode(' ', $keywords); + + $this->must_contain_ids = []; + $this->must_not_contain_ids = []; + $this->must_exclude_one_ids = []; + + foreach ($query as $word) + { + if (empty($word)) + { + continue; + } + + // words which should not be included + if ($word[0] == '-') + { + $word = substr($word, 1); + + // a group of which at least one may not be in the resulting posts + if ($word[0] == '(') + { + $word = array_unique(explode('|', substr($word, 1, -1))); + $mode = 'must_exclude_one'; + } + // one word which should not be in the resulting posts + else + { + $mode = 'must_not_contain'; + } + $ignore_no_id = true; + } + // words which have to be included + else + { + // no prefix is the same as a +prefix + if ($word[0] == '+') + { + $word = substr($word, 1); + } + + // a group of words of which at least one word should be in every resulting post + if (isset($word[0]) && $word[0] == '(') + { + $word = array_unique(explode('|', substr($word, 1, -1))); + } + $ignore_no_id = false; + $mode = 'must_contain'; + } + + if (empty($word)) + { + continue; + } + + // if this is an array of words then retrieve an id for each + if (is_array($word)) + { + $non_common_words = []; + $id_words = []; + foreach ($word as $i => $word_part) + { + if (strpos($word_part, '*') !== false) + { + $len = utf8_strlen(str_replace('*', '', $word_part)); + if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) + { + $id_words[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word_part)) . '\''; + $non_common_words[] = $word_part; + } + else + { + $this->common_words[] = $word_part; + } + } + else if (isset($words[$word_part])) + { + $id_words[] = $words[$word_part]; + $non_common_words[] = $word_part; + } + else + { + $len = utf8_strlen($word_part); + if ($len < $this->word_length['min'] || $len > $this->word_length['max']) + { + $this->common_words[] = $word_part; + } + } + } + if (count($id_words)) + { + sort($id_words); + if (count($id_words) > 1) + { + $this->{$mode . '_ids'}[] = $id_words; + } + else + { + $mode = ($mode === 'must_exclude_one') ? 'must_not_contain' : $mode; + $this->{$mode . '_ids'}[] = $id_words[0]; + } + } + // throw an error if we shall not ignore unexistant words + else if (!$ignore_no_id && count($non_common_words)) + { + trigger_error(sprintf($this->language->lang('WORDS_IN_NO_POST'), implode($this->language->lang('COMMA_SEPARATOR'), $non_common_words))); + } + unset($non_common_words); + } + // else we only need one id + else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word])) + { + if ($wildcard) + { + $len = utf8_strlen(str_replace('*', '', $word)); + if ($len >= $this->word_length['min'] && $len <= $this->word_length['max']) + { + $this->{$mode . '_ids'}[] = '\'' . $this->db->sql_escape(str_replace('*', '%', $word)) . '\''; + } + else + { + $this->common_words[] = $word; + } + } + else + { + $this->{$mode . '_ids'}[] = $words[$word]; + } + } + else + { + if (!isset($common_ids[$word])) + { + $len = utf8_strlen($word); + if ($len < $this->word_length['min'] || $len > $this->word_length['max']) + { + $this->common_words[] = $word; + } + } + } + } + + // Return true if all words are not common words + if (count($exact_words) - count($this->common_words) > 0) + { + return true; + } + return false; + } + + + /** + * {@inheritdoc} + */ + public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No keywords? No posts. + // we can't search for negatives only + if (empty($this->search_query) || empty($this->must_contain_ids)) + { + return false; + } + + $must_contain_ids = $this->must_contain_ids; + $must_not_contain_ids = $this->must_not_contain_ids; + $must_exclude_one_ids = $this->must_exclude_one_ids; + + sort($must_contain_ids); + sort($must_not_contain_ids); + sort($must_exclude_one_ids); + + // generate a search_key from all the options to identify the results + $search_key_array = [ + serialize($must_contain_ids), + serialize($must_not_contain_ids), + serialize($must_exclude_one_ids), + $type, + $fields, + $terms, + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + '', + implode(',', $author_ary), + $author_name, + ]; + $search_key = md5(implode('#', $search_key_array)); + + // try reading the results from cache + $total_results = 0; + if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $total_results; + } + + $id_ary = []; + + $sql_where = []; + $m_num = 0; + $w_num = 0; + + $sql_array = [ + 'SELECT' => 'p.article_id', + 'FROM' => [ + $this->wordmatch_table => [], + $this->wordlist_table => [], + ], + 'LEFT_JOIN' => [[ + 'FROM' => [$this->articles_table => 'p'], + 'ON' => 'm0.article_id = p.article_id', + ]], + ]; + + $title_match = ''; + $group_by = true; + // Build some display specific sql strings + switch ($fields) + { + case 'titleonly': + $title_match = 'title_match = 1 AND descr_match = 0'; + $group_by = false; + break; + + case 'msgonly': + $title_match = 'title_match = 0 AND descr_match = 0'; + $group_by = false; + break; + + case 'descronly': + $title_match = 'title_match = 0 AND descr_match = 1'; + $group_by = false; + break; + } + + foreach ($this->must_contain_ids as $subquery) + { + if (is_array($subquery)) + { + $group_by = true; + + $word_id_sql = []; + $word_ids = []; + foreach ($subquery as $id) + { + if (is_string($id)) + { + $sql_array['LEFT_JOIN'][] = [ + 'FROM' => [$this->wordlist_table => 'w' . $w_num], + 'ON' => "w$w_num.word_text LIKE $id", + ]; + $word_ids[] = "w$w_num.word_id"; + + $w_num++; + } + else + { + $word_ids[] = $id; + } + } + + $sql_where[] = $this->db->sql_in_set("m$m_num.word_id", $word_ids); + + unset($word_id_sql); + unset($word_ids); + } + else if (is_string($subquery)) + { + $sql_array['FROM'][$this->wordlist_table][] = 'w' . $w_num; + + $sql_where[] = "w$w_num.word_text LIKE $subquery"; + $sql_where[] = "m$m_num.word_id = w$w_num.word_id"; + + $group_by = true; + $w_num++; + } + else + { + $sql_where[] = "m$m_num.word_id = $subquery"; + } + + $sql_array['FROM'][$this->wordmatch_table][] = 'm' . $m_num; + + if ($title_match) + { + $sql_where[] = "m$m_num.$title_match"; + } + + if ($m_num !== 0) + { + $sql_where[] = "m$m_num.article_id = m0.article_id"; + } + $m_num++; + } + + foreach ($this->must_not_contain_ids as $key => $subquery) + { + if (is_string($subquery)) + { + $sql_array['LEFT_JOIN'][] = [ + 'FROM' => [$this->wordlist_table => 'w' . $w_num], + 'ON' => "w$w_num.word_text LIKE $subquery", + ]; + + $this->must_not_contain_ids[$key] = "w$w_num.word_id"; + + $group_by = true; + $w_num++; + } + } + + if (count($this->must_not_contain_ids)) + { + $sql_array['LEFT_JOIN'][] = [ + 'FROM' => [$this->wordmatch_table => 'm' . $m_num], + 'ON' => $this->db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.article_id = m0.article_id", + ]; + + $sql_where[] = "m$m_num.word_id IS NULL"; + $m_num++; + } + + foreach ($this->must_exclude_one_ids as $ids) + { + $is_null_joins = []; + foreach ($ids as $id) + { + if (is_string($id)) + { + $sql_array['LEFT_JOIN'][] = [ + 'FROM' => [$this->wordlist_table => 'w' . $w_num], + 'ON' => "w$w_num.word_text LIKE $id", + ]; + $id = "w$w_num.word_id"; + + $group_by = true; + $w_num++; + } + + $sql_array['LEFT_JOIN'][] = [ + 'FROM' => [$this->wordmatch_table => 'm' . $m_num], + 'ON' => "m$m_num.word_id = $id AND m$m_num.article_id = m0.article_id" . (($title_match) ? " AND m$m_num.$title_match" : ''), + ]; + $is_null_joins[] = "m$m_num.word_id IS NULL"; + + $m_num++; + } + $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')'; + } + + $sql_where[] = 'p.approved = 1'; + + $search_query = $this->search_query; + $must_exclude_one_ids = $this->must_exclude_one_ids; + $must_not_contain_ids = $this->must_not_contain_ids; + $must_contain_ids = $this->must_contain_ids; + + $sql_sort_table = $sql_sort_join = $sql_match = $sql_match_where = $sql_sort = ''; + + if ($category_id) + { + $sql_where[] = 'p.article_category_id = ' . (int) $category_id; + } + + if (count($author_ary)) + { + $sql_author = $this->db->sql_in_set('p.author_id', $author_ary); + $sql_where[] = $sql_author; + } + + if (count($ex_fid_ary)) + { + $sql_where[] = $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true); + } + + if ($sort_days) + { + $sql_where[] = 'p.article_date >= ' . (time() - ($sort_days * 86400)); + } + + $sql_array['WHERE'] = implode(' AND ', $sql_where); + + $is_mysql = false; + // if the total result count is not cached yet, retrieve it from the db + if (!$total_results) + { + $sql = ''; + $sql_array_count = $sql_array; + + switch ($this->db->get_sql_layer()) + { + case 'mysqli': + $is_mysql = true; + + break; + + case 'sqlite3': + $sql_array_count['SELECT'] = 'DISTINCT p.article_id'; + $sql = 'SELECT COUNT(article_id) as total_results + FROM (' . $this->db->sql_build_query('SELECT', $sql_array_count) . ')'; + + // no break + + default: + $sql_array_count['SELECT'] = 'COUNT(DISTINCT p.article_id) AS total_results'; + $sql = (!$sql) ? $this->db->sql_build_query('SELECT', $sql_array_count) : $sql; + + $result = $this->db->sql_query($sql); + $total_results = (int) $this->db->sql_fetchfield('total_results'); + $this->db->sql_freeresult($result); + + if (!$total_results) + { + return false; + } + break; + } + + unset($sql_array_count, $sql); + } + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + $sql_array['WHERE'] = implode(' AND ', $sql_where); + $sql_array['GROUP_BY'] = ($group_by) ? 'p.article_id, ' . $sort_by_sql[$sort_key] : ''; + $sql_array['ORDER_BY'] = $sql_sort; + $sql_array['SELECT'] .= $sort_by_sql[$sort_key] ? ", $sort_by_sql[$sort_key]" : ''; + + unset($sql_where, $sql_sort, $group_by); + + $sql = $this->db->sql_build_query('SELECT', $sql_array); + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row['article_id']; + } + $this->db->sql_freeresult($result); + + // If using mysql and the total result count is not calculated yet, get it from the db + if (!$total_results && $is_mysql) + { + $sql_count = str_replace("SELECT {$sql_array['SELECT']}", "SELECT COUNT(DISTINCT {$sql_array['SELECT']}) as total_results", $sql); + $result = $this->db->sql_query($sql_count); +// $total_results = (int) $this->db->sql_fetchfield('total_results'); + $total_results = count($this->db->sql_fetchrowset($result)); + $this->db->sql_freeresult($result); + + if (!$total_results) + { + return false; + } + } + + if ($start >= $total_results) + { + $start = floor(($total_results - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row['article_id']; + } + $this->db->sql_freeresult($result); + } + + // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page + $this->save_ids($search_key, $this->search_query, $author_ary, $total_results, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $total_results; + } + + /** + * {@inheritdoc} + */ + public function author_search(string $type, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No author? No posts + if (!count($author_ary)) + { + return 0; + } + + // generate a search_key from all the options to identify the results + $search_key_array = [ + '', + $type, + '', + '', + '', + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + '', + implode(',', $author_ary), + $author_name, + ]; + $search_key = md5(implode('#', $search_key_array)); + + // try reading the results from cache + $total_results = 0; + if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $total_results; + } + + $id_ary = []; + + // Create some display specific sql strings + $sql_author = $this->db->sql_in_set('p.author_id', $author_ary); + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true) : ''; + $sql_time = ($sort_days) ? ' AND p.article_date >= ' . (time() - ($sort_days * 86400)) : ''; + $sql_category_id = ''; + $post_visibility = ' AND p.approved = 1 '; + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + $select = 'p.article_id'; + $select .= $sort_by_sql[$sort_key] ? ", $sort_by_sql[$sort_key]" : ''; + $is_mysql = false; + + // If the cache was completely empty count the results + if (!$total_results) + { + switch ($this->db->get_sql_layer()) + { + case 'mysqli': + $is_mysql = true; + break; + + default: + $sql = 'SELECT COUNT(p.article_id) as total_results + FROM ' . $this->articles_table . ' p + WHERE ' . $sql_author . + $sql_category_id . + $sql_fora . + $sql_time . + $post_visibility; + $result = $this->db->sql_query($sql); + + $total_results = (int) $this->db->sql_fetchfield('total_results'); + $this->db->sql_freeresult($result); + + if (!$total_results) + { + return false; + } + break; + } + } + + // Build the query for really selecting the article_ids + $sql = "SELECT $select + FROM " . $this->articles_table . " p + WHERE $sql_author + $sql_category_id + $sql_fora + $sql_time + ORDER BY $sql_sort"; + $field = 'article_id'; + + // Only read one block of posts from the db and then cache it + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + if (!$total_results && $is_mysql) + { + $sql_count = str_replace("SELECT $select", 'SELECT COUNT(*) as total_results', $sql); + $result = $this->db->sql_query($sql_count); + $total_results = (int) $this->db->sql_fetchfield('total_results'); + $this->db->sql_freeresult($result); + + if (!$total_results) + { + return false; + } + } + + if ($start >= $total_results) + { + $start = floor(($total_results - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + } + + if (count($id_ary)) + { + $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $total_results; + } + return false; + } + + /** + * {@inheritdoc} + */ + public function supports_phrase_search(): bool + { + return false; + } + + /** + * {@inheritdoc} + */ + public function index(string $mode, int $article_id, string &$message, string &$subject, string &$description, int $poster_id) + { + if (!$this->config['wiki_search']) + { + /** + * The search indexer is disabled, return + */ + return; + } + + // Split old and new article/title/description to obtain array of 'words' + $split_text = $this->split_message($message); + $split_title = $this->split_message($subject); + $split_descr = $this->split_message($description); + + $cur_words = ['post' => [], 'title' => [], 'descr' => []]; + + $words = []; + if ($mode == 'edit') + { + $sql = 'SELECT w.word_id, w.word_text, m.title_match, m.descr_match + FROM ' . $this->wordlist_table . ' w, ' . $this->wordmatch_table . ' m + WHERE m.article_id = ' . $article_id . ' + AND w.word_id = m.word_id'; + $result = $this->db->sql_query($sql); + + while ($row = $this->db->sql_fetchrow($result)) + { + $which = ($row['title_match']) ? 'title' : 'post'; + $which = ($row['descr_match']) ? 'descr' : $which; + $cur_words[$which][$row['word_text']] = $row['word_id']; + } + $this->db->sql_freeresult($result); + + $words['add']['post'] = array_diff($split_text, array_keys($cur_words['post'])); + $words['add']['title'] = array_diff($split_title, array_keys($cur_words['title'])); + $words['add']['descr'] = array_diff($split_descr, array_keys($cur_words['descr'])); + $words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text); + $words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title); + $words['del']['descr'] = array_diff(array_keys($cur_words['descr']), $split_descr); + } + else + { + $words['add']['post'] = $split_text; + $words['add']['title'] = $split_title; + $words['add']['descr'] = $split_descr; + $words['del']['post'] = []; + $words['del']['title'] = []; + $words['del']['descr'] = []; + } + unset($split_text); + unset($split_title); + unset($split_descr); + + // Get unique words from the above arrays + $unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['add']['descr'])); + + // We now have unique arrays of all words to be added and removed and + // individual arrays of added and removed words for text and title. What + // we need to do now is add the new words (if they don't already exist) + // and then add (or remove) matches between the words and this post + if (count($unique_add_words)) + { + $sql = 'SELECT word_id, word_text + FROM ' . $this->wordlist_table . ' + WHERE ' . $this->db->sql_in_set('word_text', $unique_add_words); + $result = $this->db->sql_query($sql); + + $word_ids = []; + while ($row = $this->db->sql_fetchrow($result)) + { + $word_ids[$row['word_text']] = $row['word_id']; + } + $this->db->sql_freeresult($result); + $new_words = array_diff($unique_add_words, array_keys($word_ids)); + + $this->db->sql_transaction('begin'); + if (count($new_words)) + { + $sql_ary = []; + + foreach ($new_words as $word) + { + $sql_ary[] = ['word_text' => (string) $word, 'word_count' => 0]; + } + $this->db->sql_return_on_error(true); + $this->db->sql_multi_insert($this->wordlist_table, $sql_ary); + $this->db->sql_return_on_error(false); + } + unset($new_words, $sql_ary); + } + else + { + $this->db->sql_transaction('begin'); + } + + // now update the search match table, remove links to removed words and add links to new words + foreach ($words['del'] as $word_in => $word_ary) + { + $title_match = ($word_in == 'title') ? 1 : 0; + $descr_match = ($word_in == 'descr') ? 1 : 0; + + if (count($word_ary)) + { + $sql_in = []; + foreach ($word_ary as $word) + { + $sql_in[] = $cur_words[$word_in][$word]; + } + + $sql = 'DELETE FROM ' . $this->wordmatch_table . ' + WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' + AND article_id = ' . $article_id . ' + AND (descr_match = ' . $descr_match . ' + OR title_match = ' . $title_match . ')'; + $this->db->sql_query($sql); + + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_count = word_count - 1 + WHERE ' . $this->db->sql_in_set('word_id', $sql_in) . ' + AND word_count > 0'; + $this->db->sql_query($sql); + + unset($sql_in); + } + } + + $this->db->sql_return_on_error(true); + foreach ($words['add'] as $word_in => $word_ary) + { + $title_match = ($word_in == 'title') ? 1 : 0; + $descr_match = ($word_in == 'descr') ? 1 : 0; + + if (count($word_ary)) + { + $sql = 'INSERT INTO ' . $this->wordmatch_table . ' (article_id, word_id, title_match, descr_match) + SELECT ' . $article_id . ', word_id, ' . $title_match . ', ' . $descr_match . ' + FROM ' . $this->wordlist_table . ' + WHERE ' . $this->db->sql_in_set('word_text', $word_ary); + $this->db->sql_query($sql); + + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_count = word_count + 1 + WHERE ' . $this->db->sql_in_set('word_text', $word_ary); + $this->db->sql_query($sql); + } + } + $this->db->sql_return_on_error(false); + + $this->db->sql_transaction('commit'); + + // destroy cached search results containing any of the words removed or added + $this->destroy_cache(array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['add']['descr'], $words['del']['post'], $words['del']['title'], $words['del']['descr'])), [$poster_id]); + + unset($unique_add_words); + unset($words); + unset($cur_words); + } + + /** + * {@inheritdoc} + */ + public function index_remove(array $article_ids, array $author_ids): void + { + if (count([$article_ids])) + { + $sql = 'SELECT w.word_id, w.word_text, m.title_match, m.descr_match + FROM ' . $this->wordmatch_table . ' m, ' . $this->wordlist_table . ' w + WHERE ' . $this->db->sql_in_set('m.article_id', $article_ids) . ' + AND w.word_id = m.word_id'; + $result = $this->db->sql_query($sql); + + $message_word_ids = $title_word_ids = $descr_word_ids = $word_texts = []; + while ($row = $this->db->sql_fetchrow($result)) + { + if ($row['title_match']) + { + $title_word_ids[] = $row['word_id']; + } + if ($row['descr_match']) + { + $descr_word_ids[] = $row['word_id']; + } + else + { + $message_word_ids[] = $row['word_id']; + } + $word_texts[] = $row['word_text']; + } + $this->db->sql_freeresult($result); + + if (count($title_word_ids)) + { + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_count = word_count - 1 + WHERE ' . $this->db->sql_in_set('word_id', $title_word_ids) . ' + AND word_count > 0'; + $this->db->sql_query($sql); + } + + if (count($descr_word_ids)) + { + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_count = word_count - 1 + WHERE ' . $this->db->sql_in_set('word_id', $descr_word_ids) . ' + AND word_count > 0'; + $this->db->sql_query($sql); + } + + if (count($message_word_ids)) + { + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_count = word_count - 1 + WHERE ' . $this->db->sql_in_set('word_id', $message_word_ids) . ' + AND word_count > 0'; + $this->db->sql_query($sql); + } + + unset($title_word_ids); + unset($descr_word_ids); + unset($message_word_ids); + + $sql = 'DELETE FROM ' . $this->wordmatch_table . ' + WHERE ' . $this->db->sql_in_set('article_id', $article_ids); + $this->db->sql_query($sql); + } + + $this->destroy_cache(array_unique($word_texts), array_unique($author_ids)); + } + + /** + * {@inheritdoc} + */ + public function tidy(): void + { + // Is the fulltext indexer disabled? If yes then we need not + // carry on ... it's okay ... I know when I'm not wanted boo hoo + if (!$this->config['wiki_search']) + { + $this->config->set('search_last_gc', time(), false); + return; + } + + $destroy_cache_words = []; + + // Remove common words + if ($this->config['wiki_num_articles'] >= 100 && $this->config['fulltext_native_common_thres']) + { + $common_threshold = ((float) $this->config['fulltext_native_common_thres']) / 100.0; + // First, get the IDs of common words + $sql = 'SELECT word_id, word_text + FROM ' . $this->wordlist_table . ' + WHERE word_count > ' . floor($this->config['wiki_num_articles'] * $common_threshold) . ' + OR word_common = 1'; + $result = $this->db->sql_query($sql); + + $sql_in = []; + while ($row = $this->db->sql_fetchrow($result)) + { + $sql_in[] = $row['word_id']; + $destroy_cache_words[] = $row['word_text']; + } + $this->db->sql_freeresult($result); + + if (count($sql_in)) + { + // Flag the words + $sql = 'UPDATE ' . $this->wordlist_table . ' + SET word_common = 1 + WHERE ' . $this->db->sql_in_set('word_id', $sql_in); + $this->db->sql_query($sql); + + // by setting search_last_gc to the new time here we make sure that if a user reloads because the + // following query takes too long, he won't run into it again + $this->config->set('search_last_gc', time(), false); + + // Delete the matches + $sql = 'DELETE FROM ' . $this->wordmatch_table . ' + WHERE ' . $this->db->sql_in_set('word_id', $sql_in); + $this->db->sql_query($sql); + } + unset($sql_in); + } + + if (count($destroy_cache_words)) + { + // destroy cached search results containing any of the words that are now common or were removed + $this->destroy_cache(array_unique($destroy_cache_words)); + } + + $this->config->set('search_last_gc', time(), false); + } + + // create_index is inherited from base.php + + /** + * {@inheritdoc} + */ + public function delete_index(int|null &$post_counter = null): array|null + { + $sql_queries = []; + + switch ($this->db->get_sql_layer()) + { + case 'sqlite3': + $this->db->sql_query('DELETE FROM ' . $this->wordlist_table); + $this->db->sql_query('DELETE FROM ' . $this->wordmatch_table); + $this->db->sql_query('DELETE FROM ' . $this->search_results_table); + break; + + default: + $this->db->sql_query('TRUNCATE TABLE ' . $this->wordlist_table); + $this->db->sql_query('TRUNCATE TABLE ' . $this->wordmatch_table); + $this->db->sql_query('TRUNCATE TABLE ' . $this->search_results_table); + break; + } + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } + + return null; + } + + /** + * {@inheritdoc} + */ + public function index_created(): bool + { + if (!count($this->stats)) + { + $this->get_stats(); + } + + return $this->stats['total_words'] && $this->stats['total_matches']; + } + + /** + * {@inheritdoc} + */ + public function index_stats() + { + if (!count($this->stats)) + { + $this->get_stats(); + } + + return [ + $this->language->lang('TOTAL_WORDS') => $this->stats['total_words'], + $this->language->lang('TOTAL_MATCHES') => $this->stats['total_matches']]; + } + + /** + * Computes the stats and store them in the $this->stats associative array + */ + protected function get_stats() + { + $this->stats['total_words'] = $this->db->get_estimated_row_count($this->wordlist_table); + $this->stats['total_matches'] = $this->db->get_estimated_row_count($this->wordmatch_table); + } + + /** + * Split a text into words of a given length + * + * The text is converted to UTF-8, cleaned up, and split. Then, words that + * conform to the defined length range are returned in an array. + * + * NOTE: duplicates are NOT removed from the return array + * + * @param string $text Text to split, encoded in UTF-8 + * @return array Array of UTF-8 words + */ + public function split_message($text) + { + $match = $words = []; + + /** + * Taken from the original code + */ + // Do not index code + $match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is'; + // BBcode + $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#'; + + $min = $this->word_length['min']; + + $isset_min = $min - 1; + + /** + * Clean up the string, remove HTML tags, remove BBCodes + */ + $word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' '); + + while (strlen($word)) + { + if (strlen($word) > 255 || strlen($word) <= $isset_min) + { + /** + * Words longer than 255 bytes are ignored. This will have to be + * changed whenever we change the length of search_wordlist.word_text + * + * Words shorter than $isset_min bytes are ignored, too + */ + $word = strtok(' '); + continue; + } + + $len = utf8_strlen($word); + + /** + * Test whether the word is too short to be indexed. + * + * Note that this limit does NOT apply to CJK and Hangul + */ + if ($len < $min) + { + /** + * Note: this could be optimized. If the codepoint is lower than Hangul's range + * we know that it will also be lower than CJK ranges + */ + if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0) + && (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0) + && (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0)) + { + $word = strtok(' '); + continue; + } + } + + $words[] = $word; + $word = strtok(' '); + } + + return $words; + } + + /** + * Clean up a text to remove non-alphanumeric characters + * + * This method receives a UTF-8 string, normalizes and validates it, replaces all + * non-alphanumeric characters with strings then returns the result. + * + * Any number of "allowed chars" can be passed as a UTF-8 string in NFC. + * + * @param string $text Text to split, in UTF-8 (not normalized or sanitized) + * @param string $allowed_chars String of special chars to allow + * @param string $encoding Text encoding + * @return string Cleaned up text, only alphanumeric chars are left + */ + protected function cleanup($text, $allowed_chars = null, $encoding = 'utf-8') + { + static $conv = [], $conv_loaded = []; + $allow = []; + + // Convert the text to UTF-8 + $encoding = strtolower($encoding); + if ($encoding != 'utf-8') + { + $text = utf8_recode($text, $encoding); + } + + $utf_len_mask = [ + "\xC0" => 2, + "\xD0" => 2, + "\xE0" => 3, + "\xF0" => 4, + ]; + + /** + * Replace HTML entities and NCRs + */ + $text = htmlspecialchars_decode(utf8_decode_ncr($text), ENT_QUOTES); + + /** + * Normalize to NFC + */ + $text = \Normalizer::normalize($text); + + /** + * The first thing we do is: + * + * - convert ASCII-7 letters to lowercase + * - remove the ASCII-7 non-alpha characters + * - remove the bytes that should not appear in a valid UTF-8 string: 0xC0, + * 0xC1 and 0xF5-0xFF + * + * @todo in theory, the third one is already taken care of during normalization and those chars should have been replaced by Unicode replacement chars + */ + $sb_match = "ISTCPAMELRDOJBNHFGVWUQKYXZ\r\n\t!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\xC0\xC1\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"; + $sb_replace = 'istcpamelrdojbnhfgvwuqkyxz '; + + /** + * This is the list of legal ASCII chars, it is automatically extended + * with ASCII chars from $allowed_chars + */ + $legal_ascii = ' eaisntroludcpmghbfvq10xy2j9kw354867z'; + + /** + * Prepare an array containing the extra chars to allow + */ + if (isset($allowed_chars[0])) + { + $pos = 0; + $len = strlen($allowed_chars); + do + { + $c = $allowed_chars[$pos]; + + if ($c < "\x80") + { + /** + * ASCII char + */ + $sb_pos = strpos($sb_match, $c); + if (is_int($sb_pos)) + { + /** + * Remove the char from $sb_match and its corresponding + * replacement in $sb_replace + */ + $sb_match = substr($sb_match, 0, $sb_pos) . substr($sb_match, $sb_pos + 1); + $sb_replace = substr($sb_replace, 0, $sb_pos) . substr($sb_replace, $sb_pos + 1); + $legal_ascii .= $c; + } + + $pos++; + } + else + { + /** + * UTF-8 char + */ + $utf_len = $utf_len_mask[$c & "\xF0"]; + $allow[substr($allowed_chars, $pos, $utf_len)] = 1; + $pos += $utf_len; + } + } while ($pos < $len); + } + + $text = strtr($text, $sb_match, $sb_replace); + $ret = ''; + + $pos = 0; + $len = strlen($text); + + do + { + /** + * Do all consecutive ASCII chars at once + */ + if ($spn = strspn($text, $legal_ascii, $pos)) + { + $ret .= substr($text, $pos, $spn); + $pos += $spn; + } + + if ($pos >= $len) + { + return $ret; + } + + /** + * Capture the UTF char + */ + $utf_len = $utf_len_mask[$text[$pos] & "\xF0"]; + $utf_char = substr($text, $pos, $utf_len); + $pos += $utf_len; + + if (($utf_char >= self::UTF8_HANGUL_FIRST && $utf_char <= self::UTF8_HANGUL_LAST) + || ($utf_char >= self::UTF8_CJK_FIRST && $utf_char <= self::UTF8_CJK_LAST) + || ($utf_char >= self::UTF8_CJK_B_FIRST && $utf_char <= self::UTF8_CJK_B_LAST)) + { + /** + * All characters within these ranges are valid + * + * We separate them with a space in order to index each character + * individually + */ + $ret .= ' ' . $utf_char . ' '; + continue; + } + + if (isset($allow[$utf_char])) + { + /** + * The char is explicitly allowed + */ + $ret .= $utf_char; + continue; + } + + if (isset($conv[$utf_char])) + { + /** + * The char is mapped to something, maybe to itself actually + */ + $ret .= $conv[$utf_char]; + continue; + } + + /** + * The char isn't mapped, but did we load its conversion table? + * + * The search indexer table is split into blocks. The block number of + * each char is equal to its codepoint right-shifted for 11 bits. It + * means that out of the 11, 16 or 21 meaningful bits of a 2-, 3- or + * 4- byte sequence we only keep the leftmost 0, 5 or 10 bits. Thus, + * all UTF chars encoded in 2 bytes are in the same first block. + */ + if (isset($utf_char[2])) + { + if (isset($utf_char[3])) + { + /** + * 1111 0nnn 10nn nnnn 10nx xxxx 10xx xxxx + * 0000 0111 0011 1111 0010 0000 + */ + $idx = ((ord($utf_char[0]) & 0x07) << 7) | ((ord($utf_char[1]) & 0x3F) << 1) | ((ord($utf_char[2]) & 0x20) >> 5); + } + else + { + /** + * 1110 nnnn 10nx xxxx 10xx xxxx + * 0000 0111 0010 0000 + */ + $idx = ((ord($utf_char[0]) & 0x07) << 1) | ((ord($utf_char[1]) & 0x20) >> 5); + } + } + else + { + /** + * 110x xxxx 10xx xxxx + * 0000 0000 0000 0000 + */ + $idx = 0; + } + + /** + * Check if the required conv table has been loaded already + */ + if (!isset($conv_loaded[$idx])) + { + $conv_loaded[$idx] = 1; + $file = $this->phpbb_root_path . 'includes/utf/data/search_indexer_' . $idx . '.' . $this->php_ext; + + if (file_exists($file)) + { + $conv += include($file); + } + } + + if (isset($conv[$utf_char])) + { + $ret .= $conv[$utf_char]; + } + else + { + /** + * We add an entry to the conversion table so that we + * don't have to convert to codepoint and perform the checks + * that are above this block + */ + $conv[$utf_char] = ' '; + $ret .= ' '; + } + } while (1); + + return $ret; + } +} diff --git a/search/backend/wiki_fulltext_postgres.php b/search/backend/wiki_fulltext_postgres.php new file mode 100644 index 0000000..7f6d3aa --- /dev/null +++ b/search/backend/wiki_fulltext_postgres.php @@ -0,0 +1,795 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbbmodders\wiki\search\backend; + +use phpbb\config\config; +use phpbb\db\driver\driver_interface; +use phpbb\language\language; +use phpbb\user; +use RuntimeException; + +/** + * Fulltext search for PostgreSQL + */ +class wiki_fulltext_postgres extends wiki_base implements wiki_search_backend_interface +{ + /** + * Associative array holding index stats + * + * @var array + */ + protected $stats = []; + + /** + * Holds the words entered by user, obtained by splitting the entered query on whitespace + * + * @var array + */ + protected $split_words = []; + + /** + * Stores the tsearch query + * + * @var string + */ + protected $tsearch_query = ''; + + /** + * True if phrase search is supported. + * PostgreSQL fulltext currently doesn't support it + * + * @var boolean + */ + protected $phrase_search = false; + + /** + * @var language + */ + protected $language; + /** + * Contains tidied search query. + * Operators are prefixed in search query and common words excluded + * + * @var string + */ + protected $search_query = ''; + + /** + * Contains common words. + * Common words are words with length less/more than min/max length + * + * @var array + */ + protected $common_words = []; + + /** + * Associative array stores the min and max word length to be searched + * + * @var array + */ + protected $word_length = []; + + /** @var string */ + protected $articles_table; + + /** @var string */ + protected string $search_results_table; + + /** + * Constructor + * Creates a new \phpbb\search\backend\fulltext_postgres, which is used as a search backend + * + * @param config $config Config object + * @param driver_interface $db Database object + * @param language $language + * @param user $user User object + * @param string $articles_table Articles_table + * @param string $search_results_table Search_results_table + * @param string $phpbb_root_path Relative path to phpBB root + * @param string $phpEx PHP file extension + */ + public function __construct( + config $config, + driver_interface $db, + language $language, + user $user, + string $articles_table, + string $search_results_table, + string $phpbb_root_path, + string $phpEx + ) + { + global $cache; + + parent::__construct($cache, $config, $db, $user, $search_results_table); + $this->language = $language; + + $this->word_length = ['min' => $this->config['fulltext_postgres_min_word_len'], 'max' => $this->config['fulltext_postgres_max_word_len']]; + + $this->articles_table = $articles_table; + $this->search_results_table = $search_results_table; + + /** + * Load the UTF tools + */ + if (!function_exists('utf8_strlen')) + { + include($phpbb_root_path . 'includes/utf/utf_tools.' . $phpEx); + } + } + + /** + * {@inheritdoc} + */ + public function get_name(): string + { + return 'Wiki PostgreSQL Fulltext (test version)'; + } + + /** + * {@inheritdoc} + */ + public function is_available(): bool + { + return $this->db->get_sql_layer() == 'postgres'; + } + + /** + * {@inheritdoc} + */ + public function init() + { + if (!$this->is_available()) + { + return $this->language->lang('FULLTEXT_POSTGRES_INCOMPATIBLE_DATABASE'); + } + + return false; + } + + /** + * {@inheritdoc} + */ + public function get_search_query(): string + { + return $this->search_query; + } + + /** + * {@inheritdoc} + */ + public function get_common_words(): array + { + return $this->common_words; + } + + /** + * {@inheritdoc} + */ + public function get_word_length() + { + return $this->word_length; + } + + /** + * {@inheritdoc} + */ + public function split_keywords(string &$keywords, string $terms): bool + { + if ($terms == 'all') + { + $match = ['#\sand\s#iu', '#\sor\s#iu', '#\snot\s#iu', '#(^|\s)\+#', '#(^|\s)-#', '#(^|\s)\|#']; + $replace = [' +', ' |', ' -', ' +', ' -', ' |']; + + $keywords = preg_replace($match, $replace, $keywords); + } + + // Filter out as above + $split_keywords = preg_replace("#[\"\n\r\t]+#", ' ', trim(htmlspecialchars_decode($keywords, ENT_COMPAT))); + + // Split words + $split_keywords = preg_replace('#([^\p{L}\p{N}\'*"()])#u', '$1$1', str_replace('\'\'', '\' \'', trim($split_keywords))); + $matches = []; + preg_match_all('#(?:[^\p{L}\p{N}*"()]|^)([+\-|]?(?:[\p{L}\p{N}*"()]+\'?)*[\p{L}\p{N}*"()])(?:[^\p{L}\p{N}*"()]|$)#u', $split_keywords, $matches); + $this->split_words = $matches[1]; + + foreach ($this->split_words as $i => $word) + { + $clean_word = preg_replace('#^[+\-|"]#', '', $word); + + // check word length + $clean_len = utf8_strlen(str_replace('*', '', $clean_word)); + if (($clean_len < $this->config['fulltext_postgres_min_word_len']) || ($clean_len > $this->config['fulltext_postgres_max_word_len'])) + { + $this->common_words[] = $word; + unset($this->split_words[$i]); + } + } + + $this->search_query = ''; + $this->tsearch_query = ''; + if ($terms == 'any') + { + foreach ($this->split_words as $word) + { + if ((strpos($word, '+') === 0) || (strpos($word, '-') === 0) || (strpos($word, '|') === 0)) + { + $word = substr($word, 1); + } + $this->search_query .= $word . ' '; + $this->tsearch_query .= '|' . $word . ' '; + } + } + else + { + foreach ($this->split_words as $word) + { + if (strpos($word, '+') === 0) + { + $this->search_query .= $word . ' '; + $this->tsearch_query .= '&' . substr($word, 1) . ' '; + } + else if (strpos($word, '-') === 0) + { + $this->search_query .= $word . ' '; + $this->tsearch_query .= '&!' . substr($word, 1) . ' '; + } + else if (strpos($word, '|') === 0) + { + $this->search_query .= $word . ' '; + $this->tsearch_query .= '|' . substr($word, 1) . ' '; + } + else + { + $this->search_query .= '+' . $word . ' '; + $this->tsearch_query .= '&' . $word . ' '; + } + } + } + + $this->tsearch_query = substr($this->tsearch_query, 1); + $this->search_query = utf8_htmlspecialchars($this->search_query); + + if ($this->search_query) + { + $this->split_words = array_values($this->split_words); + sort($this->split_words); + return true; + } + return false; + } + + + /** + * {@inheritdoc} + */ + public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No keywords? No posts + // When search query contains queries like -foo + if (!$this->search_query || strpos($this->search_query, '+') === false) + { + return false; + } + + // generate a search_key from all the options to identify the results + $search_key_array = [ + implode(', ', $this->split_words), + $type, + $fields, + $terms, + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + true, + implode(',', $author_ary), + ]; + $search_key = md5(implode('#', $search_key_array)); + + if ($start < 0) + { + $start = 0; + } + + // try reading the results from cache + $result_count = 0; + if ($this->obtain_ids($search_key, $result_count, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $result_count; + } + + $id_ary = []; + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + // Build some display specific sql strings + switch ($fields) + { + case 'titleonly': + $sql_match = 'p.article_title'; + break; + + case 'descronly': + $sql_match = 'p.article_description'; + break; + + case 'msgonly': + $sql_match = 'p.article_body'; + break; + + default: + $sql_match = 'p.article_title, p.article_body, p.article_description'; + break; + } + + $tsearch_query = $this->tsearch_query; + + $sql_select = 'p.article_id'; + $field = 'article_id'; + + if (count($author_ary) && $author_name) + { + // first one matches post of registered users, second one guests and deleted users + $sql_author = ' AND (' . $this->db->sql_in_set('p.author_id', array_diff($author_ary, [ANONYMOUS]), false, true) . " OR p.author = '" . $this->db->sql_escape($author_name) . "')"; + } + else if (count($author_ary)) + { + $sql_author = ' AND ' . $this->db->sql_in_set('p.author_id', $author_ary); + } + else + { + $sql_author = ''; + } + + $sql_where_options = ''; + $sql_where_options .= (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true) : ''; + $sql_where_options .= $sql_author; + $sql_where_options .= ($sort_days) ? ' AND p.article_date >= ' . (time() - ($sort_days * 86400)) : ''; + $sql_where_options .= ' AND p.approved = 1 '; + + $sql_match = str_replace(',', " || ' ' ||", $sql_match); + $tmp_sql_match = "to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', " . $sql_match . ") @@ to_tsquery ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', '" . $this->db->sql_escape($this->tsearch_query) . "')"; + + $this->db->sql_transaction('begin'); + + $sql_from = 'FROM ' . $this->articles_table . ' p'; + $sql_where = 'WHERE (' . $tmp_sql_match . ") + $sql_where_options"; + $sql = "SELECT $sql_select + $sql_from + $sql_where + ORDER BY $sql_sort"; + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + + // if the total result count is not cached yet, retrieve it from the db + if (!$result_count) + { + $sql_count = "SELECT COUNT(*) as result_count + $sql_from + $sql_where"; + $result = $this->db->sql_query($sql_count); + $result_count = (int) $this->db->sql_fetchfield('result_count'); + $this->db->sql_freeresult($result); + + if (!$result_count) + { + $this->db->sql_transaction('commit'); + return false; + } + } + + $this->db->sql_transaction('commit'); + + if ($start >= $result_count) + { + $start = floor(($result_count - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + } + + // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page + $this->save_ids($search_key, implode(' ', $this->split_words), $author_ary, $result_count, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $result_count; + } + + /** + * {@inheritdoc} + */ + public function author_search(string $type, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page) + { + // No author? No posts + if (!count($author_ary)) + { + return 0; + } + + // generate a search_key from all the options to identify the results + $search_key_array = [ + '', + $type, + '', + '', + '', + $sort_days, + $sort_key, + $category_id, + implode(',', $ex_fid_ary), + true, + implode(',', $author_ary), + $author_name, + ]; + $search_key = md5(implode('#', $search_key_array)); + + if ($start < 0) + { + $start = 0; + } + + // try reading the results from cache + $result_count = 0; + if ($this->obtain_ids($search_key, $result_count, $id_ary, $start, $per_page, $sort_dir) == self::WIKI_SEARCH_RESULT_IN_CACHE) + { + return $result_count; + } + + $id_ary = []; + + // Create some display specific sql strings + $sql_author = $this->db->sql_in_set('p.author_id', $author_ary) . ' AND p.approved=1 '; + + $sql_fora = (count($ex_fid_ary)) ? ' AND ' . $this->db->sql_in_set('p.article_category_id', $ex_fid_ary, true) : ''; + $sql_category_id = ''; + $sql_time = ($sort_days) ? ' AND p.article_date >= ' . (time() - ($sort_days * 86400)) : ''; + + // Build sql strings for sorting + $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC'); + + // Build the query for really selecting the post_ids + $sql = 'SELECT p.article_id + FROM ' . $this->articles_table . ' p' . " + WHERE $sql_author + $sql_category_id + $sql_fora + $sql_time + ORDER BY $sql_sort"; + $field = 'article_id'; + + $this->db->sql_transaction('begin'); + + // Only read one block of posts from the db and then cache it + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = $row[$field]; + } + $this->db->sql_freeresult($result); + + // retrieve the total result count if needed + if (!$result_count) + { + $sql_count = 'SELECT COUNT(*) as result_count + FROM ' . $this->articles_table . " p + WHERE $sql_author + $sql_category_id + $sql_fora + $sql_time"; + + $result = $this->db->sql_query($sql_count); + $result_count = (int) $this->db->sql_fetchfield('result_count'); + $this->db->sql_freeresult($result); + + if (!$result_count) + { + $this->db->sql_transaction('commit'); + return false; + } + } + + $this->db->sql_transaction('commit'); + + if ($start >= $result_count) + { + $start = floor(($result_count - 1) / $per_page) * $per_page; + + $result = $this->db->sql_query_limit($sql, $this->config['search_block_size'], $start); + while ($row = $this->db->sql_fetchrow($result)) + { + $id_ary[] = (int) $row[$field]; + } + $this->db->sql_freeresult($result); + + $id_ary = array_unique($id_ary); + } + + if (count($id_ary)) + { + $this->save_ids($search_key, '', $author_ary, $result_count, $id_ary, $start, $sort_dir); + $id_ary = array_slice($id_ary, 0, $per_page); + + return $result_count; + } + return false; + } + + /** + * {@inheritdoc} + */ + public function supports_phrase_search(): bool + { + return $this->phrase_search; + } + + /** + * {@inheritdoc} + */ + public function index(string $mode, int $article_id, string &$message, string &$subject, string &$description, int $poster_id) + { + // Split old and new post/subject to obtain array of words + $split_text = $this->split_message($message); + $split_title = ($subject) ? $this->split_message($subject) : []; + $split_descr = ($description) ? $this->split_message($description) : []; + + $words = array_unique(array_merge($split_text, $split_title, $split_descr)); + + unset($split_text, $split_title, $split_descr); + + // destroy cached search results containing any of the words removed or added + $this->destroy_cache($words, [$poster_id]); + + unset($words); + } + + /** + * {@inheritdoc} + */ + public function index_remove(array $article_ids, array $author_ids): void + { + $this->destroy_cache([], $author_ids); + } + + /** + * {@inheritdoc} + */ + public function tidy(): void + { + // destroy too old cached search results + $this->destroy_cache([]); + + $this->config->set('search_last_gc', time(), false); + } + + /** + * {@inheritdoc} + */ + public function create_index(int &$post_counter = 0): array|null + { + // Make sure we can actually use PostgreSQL with fulltext indexes + if ($error = $this->init()) + { + throw new RuntimeException($error); + } + + if (empty($this->stats)) + { + $this->get_stats(); + } + + $sql_queries = []; + + if (!isset($this->stats['article_title'])) + { + $sql_queries[] = 'CREATE INDEX ' . $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_title ON ' . $this->articles_table . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', article_title))"; + } + + if (!isset($this->stats['article_description'])) + { + $sql_queries[] = 'CREATE INDEX ' . $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_description ON ' . $this->articles_table . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', article_description))"; + } + + if (!isset($this->stats['article_body'])) + { + $sql_queries[] = 'CREATE INDEX ' . $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_body ON ' . $this->articles_table . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', article_body))"; + } + + if (!isset($this->stats['article_content'])) + { + $sql_queries[] = 'CREATE INDEX ' . $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_content ON ' . $this->articles_table . " USING gin (to_tsvector ('" . $this->db->sql_escape($this->config['fulltext_postgres_ts_name']) . "', article_title || ' ' || article_body || ' ' || article_description))"; + } + + $stats = $this->stats; + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } + + $this->stats = []; + $this->db->sql_query('TRUNCATE TABLE ' . $this->search_results_table); + + return null; + } + + /** + * {@inheritdoc} + */ + public function delete_index(int|null &$post_counter = null): array|null + { + // Make sure we can actually use PostgreSQL with fulltext indexes + if ($error = $this->init()) + { + throw new RuntimeException($error); + } + + if (empty($this->stats)) + { + $this->get_stats(); + } + + $sql_queries = []; + + if (isset($this->stats['article_title'])) + { + $sql_queries[] = 'DROP INDEX ' . $this->stats['article_title']['relname']; + } + + if (isset($this->stats['article_description'])) + { + $sql_queries[] = 'DROP INDEX ' . $this->stats['article_description']['relname']; + } + + if (isset($this->stats['article_body'])) + { + $sql_queries[] = 'DROP INDEX ' . $this->stats['article_body']['relname']; + } + + if (isset($this->stats['article_content'])) + { + $sql_queries[] = 'DROP INDEX ' . $this->stats['article_content']['relname']; + } + + $stats = $this->stats; + + foreach ($sql_queries as $sql_query) + { + $this->db->sql_query($sql_query); + } + + $this->stats = []; + $this->db->sql_query('TRUNCATE TABLE ' . $this->search_results_table); + + return null; + } + + /** + * {@inheritdoc} + */ + public function index_created(): bool + { + if (empty($this->stats)) + { + $this->get_stats(); + } + + return isset($this->stats['article_title']) && isset($this->stats['article_content']) && isset($this->stats['article_description']) && isset($this->stats['article_body']); + } + + /** + * {@inheritdoc} + */ + public function index_stats() + { + if (empty($this->stats)) + { + $this->get_stats(); + } + + return [ + $this->language->lang('FULLTEXT_POSTGRES_TOTAL_POSTS') => ($this->index_created()) ? $this->stats['total_posts'] : 0, + ]; + } + + protected function get_stats() + { + if ($this->db->get_sql_layer() != 'postgres') + { + $this->stats = []; + return; + } + + $sql = "SELECT c2.relname, pg_catalog.pg_get_indexdef(i.indexrelid, 0, true) AS indexdef + FROM pg_catalog.pg_class c1, pg_catalog.pg_index i, pg_catalog.pg_class c2 + WHERE c1.relname = '" . $this->articles_table . "' + AND pg_catalog.pg_table_is_visible(c1.oid) + AND c1.oid = i.indrelid + AND i.indexrelid = c2.oid"; + $result = $this->db->sql_query($sql); + + while ($row = $this->db->sql_fetchrow($result)) + { + // deal with older PostgreSQL versions which didn't use Index_type + if (strpos($row['indexdef'], 'to_tsvector') !== false) + { + if ($row['relname'] == $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_title' || $row['relname'] == $this->articles_table . '_article_title') + { + $this->stats['article_title'] = $row; + } + else if ($row['relname'] == $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_description' || $row['relname'] == $this->articles_table . '_article_description') + { + $this->stats['article_description'] = $row; + } + else if ($row['relname'] == $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_body' || $row['relname'] == $this->articles_table . '_article_body') + { + $this->stats['article_body'] = $row; + } + else if ($row['relname'] == $this->articles_table . '_' . $this->config['fulltext_postgres_ts_name'] . '_article_content' || $row['relname'] == $this->articles_table . '_article_content') + { + $this->stats['article_content'] = $row; + } + + } + } + $this->db->sql_freeresult($result); + + $this->stats['total_posts'] = empty($this->stats) ? 0 : $this->db->get_estimated_row_count($this->articles_table); + } + + /** + * Turns text into an array of words + * + * @param string $text contains post text/subject + * @return array + */ + protected function split_message($text) + { + // Split words + $text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text))); + $matches = []; + preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches); + $text = $matches[1]; + + // remove too short or too long words + $text = array_values($text); + for ($i = 0, $n = count($text); $i < $n; $i++) + { + $text[$i] = trim($text[$i]); + if (utf8_strlen($text[$i]) < $this->config['fulltext_postgres_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_postgres_max_word_len']) + { + unset($text[$i]); + } + } + + return array_values($text); + } +} diff --git a/search/backend/wiki_search_backend_interface.php b/search/backend/wiki_search_backend_interface.php new file mode 100644 index 0000000..442da7f --- /dev/null +++ b/search/backend/wiki_search_backend_interface.php @@ -0,0 +1,207 @@ +split_words + * This function fills $this->search_query with the cleaned user search query + * + * If $terms is 'any' then the words will be extracted from the search query + * and combined with | inside brackets. They will afterwards be treated like + * an standard search query. + * + * Then it analyses the query and fills the internal arrays $must_not_contain_ids, + * $must_contain_ids and $must_exclude_one_ids which are later used by keyword_search() + * + * @param string $keywords contains the search query string as entered by the user + * @param string $terms is either 'all' (use search query as entered, default words to 'must be contained in + * post') or 'any' (find all posts containing at least one of the given words) + * @return boolean false if no valid keywords were found and otherwise true + */ + public function split_keywords(string &$keywords, string $terms): bool; + + /** + * Performs a search on keywords depending on display specific params. You have to run split_keywords() first + * + * @param string $type contains either posts or topics depending on what should be searched for + * @param string $fields contains either titleonly (topic titles should be searched), msgonly (only message + * bodies should be searched), firstpost (only subject and body of the first post + * should be searched) or all (all post bodies and subjects should be searched) + * @param string $terms is either 'all' (use query as entered, words without prefix should default to "have + * to be in field") or 'any' (ignore search query parts and just return all posts that + * contain any of the specified words) + * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query + * @param string $sort_key is the key of $sort_by_sql for the selected sorting + * @param string $sort_dir is either a or d representing ASC and DESC + * @param string $sort_days specifies the maximum amount of days a post may be old + * @param array $ex_fid_ary specifies an array of forum ids which should not be searched + * @param int $category_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be + * searched + * @param array $author_ary an array of author ids if the author should be ignored during the search the array + * is empty + * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match + * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and + * $per_page, should be ordered + * @param int $start indicates the first index of the page + * @param int $per_page number of ids each page is supposed to contain + * @return boolean|int total number of results + */ + public function keyword_search(string $type, string $fields, string $terms, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page); + + /** + * Performs a search on an author's posts without caring about message contents. Depends on display specific params + * + * @param string $type contains either posts or topics depending on what should be searched for + * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query + * @param string $sort_key is the key of $sort_by_sql for the selected sorting + * @param string $sort_dir is either a or d representing ASC and DESC + * @param string $sort_days specifies the maximum amount of days a post may be old + * @param array $ex_fid_ary specifies an array of forum ids which should not be searched + * @param int $category_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should + * be searched + * @param array $author_ary an array of author ids + * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match + * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start + * and $per_page, should be ordered + * @param int $start indicates the first index of the page + * @param int $per_page number of ids each page is supposed to contain + * @return boolean|int total number of results + */ + public function author_search(string $type, array $sort_by_sql, string $sort_key, string $sort_dir, string $sort_days, array $ex_fid_ary, int $category_id, array $author_ary, string $author_name, array &$id_ary, int &$start, int $per_page); + + /** + * Returns if phrase search is supported or not + * + * @return bool + */ + public function supports_phrase_search(): bool; + + /** + * Updates wordlist and wordmatch tables when a message is posted or changed + * Destroys cached search results, that contained one of the new words in a post so the results won't be outdated + * + * @param string $mode contains the post mode: edit, post, reply, quote ... + * @param int $article_id contains the post id of the post to index + * @param string $message contains the post text of the post + * @param string $subject contains the subject of the post to index + * @param int $poster_id contains the user id of the poster + */ + public function index(string $mode, int $article_id, string &$message, string &$subject, string &$description, int $poster_id); + + /** + * Destroy cached results, that might be outdated after deleting a post + * + * @param array $article_ids + * @param array $author_ids + * + * @return void + */ + public function index_remove(array $article_ids, array $author_ids): void; + + /** + * Destroy old cache entries + * + * @return void + */ + public function tidy(): void; + + /** + * Create fulltext index + * + * @param int $post_counter + * @return array|null array with current status or null if finished + */ + public function create_index(int &$post_counter = 0): array|null; + + /** + * Drop fulltext index + * + * @param int $post_counter + * @return array|null array with current status or null if finished + */ + public function delete_index(int &$post_counter = 0): array|null; + + /** + * Returns true if both FULLTEXT indexes exist + * + * @return bool + */ + public function index_created(): bool; + + /** + * Returns an associative array containing information about the indexes + * + * @return array|false Language string of error false otherwise + */ + public function index_stats(); + + /** + * Display various options that can be configured for the backend from the acp + * + * @return array array containing template and config variables + */ +// public function get_acp_options(): array; + + /** + * Gets backend class + * + * @return string + */ + public function get_type(): string; +} diff --git a/search/wiki_search_backend_factory.php b/search/wiki_search_backend_factory.php new file mode 100644 index 0000000..09742aa --- /dev/null +++ b/search/wiki_search_backend_factory.php @@ -0,0 +1,61 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbbmodders\wiki\search; + +use phpbb\config\config; +use phpbb\di\service_collection; +use phpbbmodders\wiki\search\backend\wiki_search_backend_interface; + +class wiki_search_backend_factory +{ + /** @var config */ + protected config $config; + + /** @var service_collection */ + protected service_collection $search_backends; + + /** + * Constructor + * + * @param config $config + * @param service_collection $search_backends + */ + public function __construct(config $config, service_collection $search_backends) + { + $this->config = $config; + $this->search_backends = $search_backends; + } + + /** + * Obtains a specified search backend + * + * @param string $class + * + * @return wiki_search_backend_interface + */ + public function get(string $class): wiki_search_backend_interface + { + return $this->search_backends->get_by_class($class); + } + + /** + * Obtains active search backend + * + * @return wiki_search_backend_interface + */ + public function get_active(): wiki_search_backend_interface + { + return $this->get($this->config['wiki_search_type']); + } +} diff --git a/styles/prosilver/template/overview.html b/styles/prosilver/template/overview.html index f641a83..b32aaad 100644 --- a/styles/prosilver/template/overview.html +++ b/styles/prosilver/template/overview.html @@ -8,6 +8,16 @@

{L_OVERVIEW}

• {L_WIKI_FEED_ATOM_LINK}

+ +
+

+ + + +

+
+ +
diff --git a/styles/prosilver/template/search.html b/styles/prosilver/template/search.html new file mode 100644 index 0000000..8840a42 --- /dev/null +++ b/styles/prosilver/template/search.html @@ -0,0 +1,84 @@ + +

{L_WIKI_SEARCH}

+ + +
+

{L_WIKI_SEARCH_NOT_BUILT} {L_WIKI_SEARCH_BUILD}

+
+ + +
+
+
+
+
+

{L_SEARCH_KEYWORDS_EXPLAIN}
+
+
+ + +
+
+
+
+
+ +
+
+
+
+
+ + +
+
+
+
 
+
+
+
+
+
+
+ + +

{SEARCH_MATCHES}

+ + +
+
+
+

{results.TITLE}

+

{results.DESCRIPTION}

+
{results.SNIPPET}
+

{results.DATE} • {results.AUTHOR}

+
+
+
+ + +

{L_WIKI_SEARCH_NO_RESULTS}

+ +
+ +
+ + + diff --git a/tests/unit/search_sync_test.php b/tests/unit/search_sync_test.php new file mode 100644 index 0000000..610215e --- /dev/null +++ b/tests/unit/search_sync_test.php @@ -0,0 +1,43 @@ +assertSame($expected, search_sync::plain_text($raw)); + } + + public static function texts(): array + { + return array( + 'plain' => array('Just some words.', 'Just some words.'), + 'bbcode' => array('A [b]bold[/b] and [url=https://example.org]a link[/url] and [list][*]one[*]two[/list]', 'A bold and a link and one two'), + 'wiki links' => array('See [[Main Page]] and [[Getting Started#Editing|the editing part]].', 'See Main Page and the editing part .'), + 'a file keeps its alt' => array('[[File:diagram.png|The release diagram]] and [[File:notes.zip]]', 'The release diagram and'), + 'categories vanish' => array('Text [[Category:Boxes]] more [[category:old_things|key]]', 'Text more'), + 'templates' => array('{{Out of date|reason=old text}} and {{{1|default}}}', 'Out of date reason=old text and 1 default'), + 'tables' => array("\n\n
Name[b]Wiki[/b]
", 'Name Wiki'), + 'headings' => array("== History ==\nText\n=== Early ===\nMore", 'History Text Early More'), + 'noinclude' => array('DocsBody', 'Docs Body'), + 'escaped characters' => array('Fish & chips <3', 'Fish & chips <3'), + 'lines and tabs' => array("one\n\n\ttwo three", 'one two three'), + 'empty' => array('', ''), + ); + } +} diff --git a/wiki/delete.php b/wiki/delete.php index ceee4bf..86bcb36 100644 --- a/wiki/delete.php +++ b/wiki/delete.php @@ -53,8 +53,9 @@ class delete extends \phpbbmodders\wiki\wiki\functions * @param \phpbb\request\request $request Request object * @param \phpbbmodders\wiki\wiki\watch $watch Watched articles */ - public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\user $user, $article_table, \phpbbmodders\wiki\wiki\files $files, \phpbb\request\request $request, \phpbbmodders\wiki\wiki\watch $watch) + public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\user $user, $article_table, \phpbbmodders\wiki\wiki\files $files, \phpbb\request\request $request, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\search_sync $search_sync) { + $this->search_sync = $search_sync; $this->files = $files; $this->request = $request; $this->watch = $watch; @@ -124,6 +125,9 @@ public function article($article) { $article = self::confirmed_article($this->request, $article); + // Out of the search first, while its versions can still be found + $this->search_sync->remove($article); + $sql = 'DELETE FROM ' . $this->article_table . " WHERE article_url = '" . $this->db->sql_escape($article) . "'"; $this->db->sql_query($sql); diff --git a/wiki/edit.php b/wiki/edit.php index 25659c2..9c76a79 100644 --- a/wiki/edit.php +++ b/wiki/edit.php @@ -78,9 +78,11 @@ class edit extends \phpbbmodders\wiki\wiki\functions * @param string $php_ext * @param \phpbbmodders\wiki\wiki\watch $watch Watched articles object * @param \phpbbmodders\wiki\wiki\templates $templates Wiki templates object + * @param \phpbbmodders\wiki\wiki\search_sync $search_sync Keeps the search current */ - public function __construct(\phpbb\auth\auth $auth, \phpbb\config\config $config, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\notification\manager $notification_manager, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, $article_table, $phpbb_root_path, $php_ext, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\templates $templates) + public function __construct(\phpbb\auth\auth $auth, \phpbb\config\config $config, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\notification\manager $notification_manager, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, $article_table, $phpbb_root_path, $php_ext, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\templates $templates, \phpbbmodders\wiki\wiki\search_sync $search_sync) { + $this->search_sync = $search_sync; $this->templates = $templates; $this->watch = $watch; $this->auth = $auth; @@ -148,6 +150,7 @@ public function deactivate($article) SET article_approved = 0 WHERE article_url = '" . $this->db->sql_escape($article) . "'"; $this->db->sql_query($sql); + $this->sync_search($article); trigger_error($this->user->lang('DEACTIVATE_ARTICLE_SUCCESS') . '

' . $this->user->lang('BACK_TO_WIKI') . ''); } diff --git a/wiki/functions.php b/wiki/functions.php index 62ecde3..352bbbe 100644 --- a/wiki/functions.php +++ b/wiki/functions.php @@ -12,6 +12,23 @@ class functions { + /** @var \phpbbmodders\wiki\wiki\search_sync|null Keeps the search current; set by the classes that change articles */ + protected $search_sync = null; + + /** + * Tell the search that the live version of an article may have changed. + * + * @param string $article_url + * @return void + */ + protected function sync_search($article_url) + { + if ($this->search_sync !== null) + { + $this->search_sync->sync($article_url); + } + } + /** @var \phpbb\auth\auth */ protected $auth; @@ -110,6 +127,8 @@ protected function set_active_version($id) SET article_approved = 1 WHERE article_id = ' . (int) $id; $this->db->sql_query($sql); + $this->sync_search($row['article_url']); + return $row['article_url']; } diff --git a/wiki/search_sync.php b/wiki/search_sync.php new file mode 100644 index 0000000..7dd3252 --- /dev/null +++ b/wiki/search_sync.php @@ -0,0 +1,251 @@ +config = $config; + $this->db = $db; + $this->factory = $factory; + $this->article_table = $article_table; + $this->search_text_table = $search_text_table; + } + + /** + * Plain text for the search from an article's text as it is written: BBCode, links, + * templates, tables, headings and categories reduced to the words they hold. + * + * @param string $raw The article text as an editor sees it + * @return string + */ + public static function plain_text($raw) + { + $text = htmlspecialchars_decode((string) $raw, ENT_QUOTES); + + $text = preg_replace('/\[\[\s*Category:[^\]]*\]\]/i', ' ', $text); // [[Category:Name]] + $text = preg_replace('/\[\[\s*File:[^\]|]*(?:\|([^\]]*))?\]\]/i', ' $1 ', $text); // [[File:name|alt]] keeps the alt text + $text = preg_replace('/\[\[[^\]|]*\|([^\]]*)\]\]/', ' $1 ', $text); // [[Article|text]] keeps the text + $text = preg_replace('/\[\[([^\]]*)\]\]/', ' $1 ', $text); // [[Article]] + $text = preg_replace('/\[\/?[a-z*][^\]\n]*\]/i', ' ', $text); // [b], [/b], [url=...], [*] + $text = preg_replace('/<[^>\n]*>/', ' ', $text); // , , ... + $text = preg_replace('/\{\{\{?|\}\}\}?|\||^={2,6}|={2,6}[ \t]*$/m', ' ', $text); // {{templates}}, bars, == headings == + $text = preg_replace('/\s+/u', ' ', $text); + + return trim((string) $text); + } + + /** + * The ids, in the search's copy, of the versions of an article. + * + * @param string $article_url + * @return array Rows with article_id and author_id + */ + protected function indexed_versions($article_url) + { + $sql = 'SELECT s.article_id, s.author_id + FROM ' . $this->search_text_table . ' s + INNER JOIN ' . $this->article_table . " a ON (a.article_id = s.article_id) + WHERE a.article_url = '" . $this->db->sql_escape($article_url) . "'"; + $result = $this->db->sql_query($sql); + $rows = $this->db->sql_fetchrowset($result); + $this->db->sql_freeresult($result); + + return $rows; + } + + /** + * Take versions out of the search's copy and out of the backend's index. + * + * @param array $rows Rows with article_id and author_id + * @return void + */ + protected function drop(array $rows) + { + if (empty($rows)) + { + return; + } + + $ids = array_map('intval', array_column($rows, 'article_id')); + $authors = array_map('intval', array_column($rows, 'author_id')); + + $this->factory->get_active()->index_remove($ids, $authors); + $this->db->sql_query('DELETE FROM ' . $this->search_text_table . ' WHERE ' . $this->db->sql_in_set('article_id', $ids)); + } + + /** + * Make the search's copy of an article match its live version: add it or update it, and + * drop the versions that are no longer live. + * + * @param string $article_url The article's name as in its address + * @return void + */ + public function sync($article_url) + { + $sql = 'SELECT a.article_id, a.article_title, a.article_description, a.article_text, a.bbcode_uid, + a.article_last_edit, a.article_user_id, u.username + FROM ' . $this->article_table . ' a + LEFT JOIN ' . USERS_TABLE . " u ON (u.user_id = a.article_user_id) + WHERE a.article_url = '" . $this->db->sql_escape($article_url) . "' + AND a.article_approved = 1 + ORDER BY a.article_last_edit DESC, a.article_id DESC"; + $result = $this->db->sql_query_limit($sql, 1); + $live = $this->db->sql_fetchrow($result); + $this->db->sql_freeresult($result); + + $known = $this->indexed_versions($article_url); + $stale = array_filter($known, function ($row) use ($live) { + return !$live || (int) $row['article_id'] !== (int) $live['article_id']; + }); + $this->drop($stale); + + if ($live) + { + $this->store($live, count($known) > count($stale)); + } + + $this->count_articles(); + } + + /** + * Take an article out of the search before it is deleted, while the versions can still be found. + * + * @param string $article_url + * @return void + */ + public function remove($article_url) + { + $this->drop($this->indexed_versions($article_url)); + $this->count_articles(); + } + + /** + * Put one live version in the search's copy and index it. + * + * @param array $live Row with the version's fields and the author's name + * @param bool $update Whether the search already holds this article + * @return void + */ + protected function store(array $live, $update) + { + $edit = generate_text_for_edit($live['article_text'], $live['bbcode_uid'], OPTION_FLAG_BBCODE | OPTION_FLAG_SMILIES | OPTION_FLAG_LINKS); + $body = self::plain_text($edit['text']); + $title = (string) $live['article_title']; + $description = (string) $live['article_description']; + $author_id = (int) $live['article_user_id']; + + $this->db->sql_query('DELETE FROM ' . $this->search_text_table . ' WHERE article_id = ' . (int) $live['article_id']); + $this->db->sql_query('INSERT INTO ' . $this->search_text_table . ' ' . $this->db->sql_build_array('INSERT', array( + 'article_id' => (int) $live['article_id'], + 'article_category_id' => 0, + 'approved' => 1, + 'article_title' => $title, + 'article_description' => $description, + 'article_date' => (int) $live['article_last_edit'], + 'author_id' => $author_id, + 'author' => (string) $live['username'], + 'article_body' => $body, + ))); + + $this->factory->get_active()->index($update ? 'edit' : 'post', (int) $live['article_id'], $body, $title, $description, $author_id); + } + + /** + * Rebuild the search's copy from every live article, and drop what no longer has an article. + * The backend's own index is built from it in the ACP. + * + * @return int How many articles the copy holds + */ + public function resync_all() + { + $this->db->sql_query('DELETE FROM ' . $this->search_text_table); + + $sql = 'SELECT a.article_id, a.article_title, a.article_description, a.article_text, a.bbcode_uid, + a.article_last_edit, a.article_user_id, u.username + FROM ' . $this->article_table . ' a + LEFT JOIN ' . USERS_TABLE . ' u ON (u.user_id = a.article_user_id) + WHERE a.article_approved = 1 + AND NOT EXISTS ( + SELECT 1 FROM ' . $this->article_table . ' b + WHERE b.article_url = a.article_url + AND b.article_approved = 1 + AND (b.article_last_edit > a.article_last_edit + OR (b.article_last_edit = a.article_last_edit AND b.article_id > a.article_id))) + ORDER BY a.article_id'; + $result = $this->db->sql_query($sql); + while ($live = $this->db->sql_fetchrow($result)) + { + $edit = generate_text_for_edit($live['article_text'], $live['bbcode_uid'], OPTION_FLAG_BBCODE | OPTION_FLAG_SMILIES | OPTION_FLAG_LINKS); + $this->db->sql_query('INSERT INTO ' . $this->search_text_table . ' ' . $this->db->sql_build_array('INSERT', array( + 'article_id' => (int) $live['article_id'], + 'article_category_id' => 0, + 'approved' => 1, + 'article_title' => (string) $live['article_title'], + 'article_description' => (string) $live['article_description'], + 'article_date' => (int) $live['article_last_edit'], + 'author_id' => (int) $live['article_user_id'], + 'author' => (string) $live['username'], + 'article_body' => self::plain_text($edit['text']), + ))); + } + $this->db->sql_freeresult($result); + + return $this->count_articles(); + } + + /** + * Keep the setting the backends read for "how many articles" up to date. + * + * @return int + */ + protected function count_articles() + { + $result = $this->db->sql_query('SELECT COUNT(article_id) AS total FROM ' . $this->search_text_table); + $total = (int) $this->db->sql_fetchfield('total'); + $this->db->sql_freeresult($result); + $this->config->set('wiki_num_articles', $total, true); + + return $total; + } +}