From f5e0fef2cb57ba5d0a6fc916d7a036f930a9b492 Mon Sep 17 00:00:00 2001 From: William Jacoby Date: Wed, 14 Oct 2026 23:05:04 -0500 Subject: [PATCH] Keep category memberships, for template categories, sub-categories and sort keys Which articles are in which category is now kept in a table, worked out from an article's rendered live version whenever it changes, and for every article that uses a template when the template changes. That lets a category list the articles a template puts in it, list the category pages that are in it as sub-categories, and sort its members by the key after the bar, which the parser now keeps. [[:Category:Name]] is a plain link to the category's page. A cron task fills the table for existing articles in batches; until then the lists work as before. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +- config/services.yml | 25 ++ controller/category.php | 10 + cron/task/category_members_build.php | 49 +++ docs/TODO.md | 14 +- event/wikilink_listener.php | 26 +- language/en/common.php | 1 + migrations/add_category_members.php | 66 ++++ .../prosilver/template/category_members.html | 8 + tests/unit/categories_test.php | 14 + tests/unit/markup_test.php | 12 +- tests/wiki/category_members_test.php | 117 +++++++ tests/wiki/fixtures/category_members.xml | 26 ++ wiki/categories.php | 83 ++++- wiki/category_members.php | 301 ++++++++++++++++++ wiki/delete.php | 10 +- wiki/edit.php | 4 +- wiki/functions.php | 9 +- wiki/markup.php | 12 + wiki/move.php | 11 +- 20 files changed, 769 insertions(+), 41 deletions(-) create mode 100644 cron/task/category_members_build.php create mode 100644 migrations/add_category_members.php create mode 100644 tests/wiki/category_members_test.php create mode 100644 tests/wiki/fixtures/category_members.xml create mode 100644 wiki/category_members.php diff --git a/README.md b/README.md index 1aba3cb..41cbdc7 100644 --- a/README.md +++ b/README.md @@ -124,12 +124,16 @@ An article Write `[[Category:Name]]` anywhere in a wiki article to put it in a category. The mark shows nothing in the text; the article lists its categories at the bottom, each linking to the category's page. -- `[[Category:boxes]]` and `[[Category:Boxes]]` are the same category: the first letter is capitalised and underscores count as spaces. A sort key after a bar (`[[Category:Boxes|key]]`) is accepted and ignored. -- **A category page** is the article `Category:Name`, at `/wiki/Category:Name`. If nobody wrote it, the page still exists as a list of the articles in the category, A to Z with their descriptions, with a link to write a description. If an article called `Category:Name` exists, its text is shown above the list, and it has the usual edit and version buttons. +- `[[Category:boxes]]` and `[[Category:Boxes]]` are the same category: the first letter is capitalised and underscores count as spaces. +- **Sort keys:** `[[Category:Boxes|Small]]` files the article under "Small" in that category's list instead of under its title. If an article names a category twice, the first key counts. +- **Linking to a category:** `[[:Category:Boxes]]`, with a colon in front, is a plain link to the category's page and does not put the article in it. +- **Sub-categories:** a category page that is itself in a category (`Category:Extensions` with `[[Category:Software]]` in it) is listed as a sub-category on that category's page, above its articles. +- **A category page** is the article `Category:Name`, at `/wiki/Category:Name`. If nobody wrote it, the page still exists as a list of the articles in the category, by sort key or title, with their descriptions, with a link to write a description. If an article called `Category:Name` exists, its text is shown above the list, and it has the usual edit and version buttons. - **`/wiki/categories`** lists every category with a live article in it, how many, and its description (the description field of its `Category:Name` article), linked from the overview. A **How to use categories** box above the list explains `[[Category:Name]]` and category pages, plus, for members who can edit, how to describe a category. Under each category, **Articles in it and how to add one** opens the line to write and its first five articles. Category pages are left out of the overview's article lists. - Only the live version of an article counts. A forum post keeps `[[Category:Name]]` as plain text, like the rest of the wiki markup. -- A template page counts for the categories it names outside ``, so `[[Category:Infobox templates]]` puts the template in that category. A category added inside `` by a template shows under the articles that use the template, but those articles are not in the category's list, which is found from the articles' own text. -- After upgrading, disable and enable the extension so the board rebuilds its cache. Articles saved before this build keep `[[Category:Name]]` as text until they are saved again or reparsed with `php bin/phpbbcli.php reparser:reparse phpbbmodders_wiki_article`. +- A template page counts for the categories it names outside ``, so `[[Category:Infobox templates]]` puts the template in that category. A category inside `` puts every article that uses the template in it, as on MediaWiki: `{{Stub}}` can put its articles in "Stubs". When a template changes, the articles that use it are worked out again. +- Which articles are in which category is kept in a table, worked out whenever an article's live version changes. After the update that adds it, a cron task fills it for the articles the board already has, a batch at a time on ordinary page views; until it is done, the category pages read the articles' text as before (without templates' categories, sub-categories or sort keys). +- After upgrading, disable and enable the extension so the board rebuilds its cache. Articles saved before sort keys keep the key they were written with only once they are saved again or reparsed. Articles saved before this build keep `[[Category:Name]]` as text until they are saved again or reparsed with `php bin/phpbbcli.php reparser:reparse phpbbmodders_wiki_article`. ## Search diff --git a/config/services.yml b/config/services.yml index 5f263b6..8a3c7d9 100644 --- a/config/services.yml +++ b/config/services.yml @@ -9,6 +9,7 @@ parameters: tables.wiki.template_install: '%core.table_prefix%wiki_template_install' tables.wiki.edit_lock: '%core.table_prefix%wiki_edit_lock' tables.wiki.article_protect: '%core.table_prefix%wiki_article_protect' + tables.wiki.category_member: '%core.table_prefix%wiki_category_member' services: _defaults: @@ -53,6 +54,7 @@ services: - '@dbal.conn' - '@phpbbmodders.wiki.templates' - '%tables.wiki.article%' + - '@phpbbmodders.wiki.category_members' phpbbmodders.wiki.category: class: phpbbmodders\wiki\controller\category @@ -164,6 +166,7 @@ services: - '%tables.wiki.file%' - '%tables.wiki.watch%' - '@phpbbmodders.wiki.protection' + - '@phpbbmodders.wiki.category_members' phpbbmodders.wiki.listener: class: phpbbmodders\wiki\event\listener arguments: @@ -195,6 +198,26 @@ services: - '@phpbbmodders.wiki.search_sync' - '@phpbbmodders.wiki.edit_lock' - '@phpbbmodders.wiki.protection' + - '@phpbbmodders.wiki.category_members' + # Which articles are in which category, kept current as articles go live, and the cron + # task that fills it for the articles a board already has + phpbbmodders.wiki.category_members: + class: phpbbmodders\wiki\wiki\category_members + arguments: + - '@config' + - '@dbal.conn' + - '@phpbbmodders.wiki.links' + - '@phpbbmodders.wiki.templates' + - '%tables.wiki.article%' + - '%tables.wiki.category_member%' + phpbbmodders.wiki.cron.task.category_members_build: + class: phpbbmodders\wiki\cron\task\category_members_build + arguments: + - '@phpbbmodders.wiki.category_members' + calls: + - [set_name, [cron.task.phpbbmodders.wiki.category_members_build]] + tags: + - { name: cron.task } # Per-article edit protection: which groups may edit an article, and the page that sets it phpbbmodders.wiki.protection: class: phpbbmodders\wiki\wiki\protection @@ -249,6 +272,8 @@ services: - '@phpbbmodders.wiki.watch' - '@phpbbmodders.wiki.search_sync' - '@phpbbmodders.wiki.protection' + - '@phpbbmodders.wiki.category_members' + - '@phpbbmodders.wiki.links' phpbbmodders.wiki.view: class: phpbbmodders\wiki\wiki\view arguments: diff --git a/controller/category.php b/controller/category.php index 29cf769..492e3af 100644 --- a/controller/category.php +++ b/controller/category.php @@ -96,6 +96,16 @@ public function __construct(\phpbb\auth\auth $auth, \phpbb\config\config $config */ protected function assign_members($name) { + // Categories that are in this one, by sort key or else name + foreach ($this->categories->subcategories($name) as $row) + { + $this->template->assign_block_vars('category_subcategories', array( + 'NAME' => substr($row['article_url'], strlen(categories::PREFIX)), + 'DESCRIPTION' => $row['article_description'], + 'U_CATEGORY' => $this->helper->route('phpbbmodders_wiki_article', array('article' => $row['article_url'])), + )); + } + $members = $this->categories->members($name); $total = count($members); $per_page = max(1, (int) $this->config['topics_per_page']); diff --git a/cron/task/category_members_build.php b/cron/task/category_members_build.php new file mode 100644 index 0000000..5740889 --- /dev/null +++ b/cron/task/category_members_build.php @@ -0,0 +1,49 @@ +category_members = $category_members; + } + + public function run() + { + $this->category_members->build_batch(); + } + + public function is_runnable() + { + return !$this->category_members->built(); + } + + /** + * Every request may take the next batch until the table is built. + * + * @return bool + */ + public function should_run() + { + return true; + } +} diff --git a/docs/TODO.md b/docs/TODO.md index ed88021..2858b35 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -25,17 +25,13 @@ Built: `[[Category:Name]]` in an article puts it in a category, a category page (`/wiki/Category:Name`) lists the articles in it, and `/wiki/categories` lists them all. See the README. +Also built (10/15/2026): categories added by a template, sub-categories, `[[:Category:Name]]` +links and sort keys, from a stored membership table that a cron task fills after the update. + Not built: -- A category added by a template (`[[Category:Stubs]]`) shows under - the articles that use the template, but those articles are not in the category's member list, - which is found from the articles' own text. Listing them needs a table of memberships that is - worked out when a version goes live and again for every article that uses a template when the - template changes. -- Sub-categories, `[[:Category:Name]]` (a link to a category without joining it), and sort keys - (a sort key after a bar is accepted and ignored). -- The member list reads every live article that holds a category mark, so it is meant for wikis - of moderate size; a membership table would also fix that. +- Paging the sub-categories of a category separately from its articles, and showing how many + articles each sub-category holds. ## A wiki-style UI, not just reskinned forum templates diff --git a/event/wikilink_listener.php b/event/wikilink_listener.php index 67230f1..74534e1 100644 --- a/event/wikilink_listener.php +++ b/event/wikilink_listener.php @@ -119,17 +119,20 @@ public function configure_wikilink($event) return; } - // [[Category:Name]] puts the article in a category and shows nothing where it stands. The - // parser reads it only while a wiki article is saved (see toggle_wiki_markup()); a sort key - // after a bar is accepted and ignored. + // [[Category:Name]] puts the article in a category and shows nothing where it stands, and + // [[Category:Name|Key]] files it under Key in that category's list. The parser reads it + // only while a wiki article is saved (see toggle_wiki_markup()). $category = $configurator->tags->add('WIKICATEGORY'); $category_name = $category->attributes->add('name'); $category_name->filterChain->append('\phpbbmodders\wiki\wiki\markup::decode_entities_in_name'); $category_name->filterChain->append('\phpbbmodders\wiki\wiki\categories::normalise'); - $category->template = ''; + $category_key = $category->attributes->add('key'); + $category_key->required = false; + $category_key->filterChain->append('\phpbbmodders\wiki\wiki\markup::decode_entities'); + $category->template = ''; $configurator->Preg->match( - '/\[\[\s*Category:\s*(?=[^\s\]|])(?' . self::NAME_CLASS . '{1,255}?)\s*(?:\|[^\[\]\n]*)?\]\]/i', + '/\[\[\s*Category:\s*(?=[^\s\]|])(?' . self::NAME_CLASS . '{1,255}?)\s*(?:\|\s*(?[^\[\]\n]{0,255}?)\s*)?\]\]/i', 'WIKICATEGORY' ); @@ -193,6 +196,8 @@ public function configure_wikilink($event) $link_attribute->required = false; $link_attribute->filterChain->append('\phpbbmodders\wiki\wiki\markup::' . $decode); } + // [[:Category:Name]] links to a category's page without putting the article in it + $tag->attributes['name']->filterChain->append('\phpbbmodders\wiki\wiki\markup::strip_link_colon'); $tag->template = '' . '' . '#'; @@ -311,15 +316,16 @@ public function link_wikilinks($event) if (strpos($html, 'class="wikicategory"') !== false) { - // Note the categories, for the article's page to list, and take the marks out - if (preg_match_all('/<\/span>/', $html, $marks)) + // Note the categories (and their sort keys), for the article's page to list and the + // category lists to file it under, and take the marks out + if (preg_match_all('/<\/span>/', $html, $marks, PREG_SET_ORDER)) { - foreach ($marks[1] as $mark) + foreach ($marks as $mark) { - \phpbbmodders\wiki\wiki\categories::add(htmlspecialchars_decode($mark, ENT_QUOTES)); + \phpbbmodders\wiki\wiki\categories::add(htmlspecialchars_decode($mark[1], ENT_QUOTES), isset($mark[2]) ? htmlspecialchars_decode($mark[2], ENT_QUOTES) : ''); } } - $html = preg_replace('/<\/span>/', '', $html); + $html = preg_replace('/<\/span>/', '', $html); $event['html'] = $html; } diff --git a/language/en/common.php b/language/en/common.php index 1650757..3fab1b6 100644 --- a/language/en/common.php +++ b/language/en/common.php @@ -176,6 +176,7 @@ 2 => '%d articles', ), 'WIKI_CATEGORY_MEMBERS' => 'Articles in this category', + 'WIKI_CATEGORY_SUBCATEGORIES' => 'Sub-categories', 'WIKI_CATEGORY_EMPTY' => 'No article is in this category yet.', 'WIKI_CATEGORY_NO_PAGE' => 'This category has no description yet.', 'WIKI_CATEGORY_WRITE' => 'Write one', diff --git a/migrations/add_category_members.php b/migrations/add_category_members.php new file mode 100644 index 0000000..947abbe --- /dev/null +++ b/migrations/add_category_members.php @@ -0,0 +1,66 @@ + array( + $this->table_prefix . 'wiki_category_member' => array( + 'COLUMNS' => array( + 'member_id' => array('UINT', null, 'auto_increment'), + 'article_url' => array('VCHAR_UNI:255', ''), + 'category' => array('VCHAR_UNI:255', ''), + 'sort_key' => array('VCHAR_UNI:255', ''), + ), + // A number for the key: two names would be too long a key for SQL Server + 'PRIMARY_KEY' => 'member_id', + 'KEYS' => array( + 'article_url' => array('INDEX', 'article_url'), + 'category' => array('INDEX', 'category'), + ), + ), + ), + ); + } + + public function revert_schema() + { + return array( + 'drop_tables' => array( + $this->table_prefix . 'wiki_category_member', + ), + ); + } + + public function update_data() + { + return array( + array('config.add', array('wiki_category_members_built', 0)), + array('config.add', array('wiki_category_members_cursor', '', true)), + ); + } +} diff --git a/styles/prosilver/template/category_members.html b/styles/prosilver/template/category_members.html index a548c8d..df4b667 100644 --- a/styles/prosilver/template/category_members.html +++ b/styles/prosilver/template/category_members.html @@ -1,3 +1,11 @@ + +

{L_WIKI_CATEGORY_SUBCATEGORIES}

+ +

{L_WIKI_CATEGORY_MEMBERS}

diff --git a/tests/unit/categories_test.php b/tests/unit/categories_test.php index 86e66af..84f7921 100644 --- a/tests/unit/categories_test.php +++ b/tests/unit/categories_test.php @@ -73,4 +73,18 @@ public function test_a_template_page_counts_for_what_it_says_outside_includeonly $this->assertSame(array(), categories::names_outside_includeonly('[[Category:Boxes]]')); $this->assertSame(array(), categories::names_outside_includeonly('No marks')); } + + public function test_sort_keys_are_kept_with_the_first_mark() + { + $raw = "[[Category:Boxes| Small box ]] [[Category:boxes|other]] [[Category:Plain]]"; + $this->assertSame(array('Boxes' => 'Small box', 'Plain' => ''), categories::keys_outside_includeonly($raw)); + + categories::reset(); + categories::add('boxes', 'B key'); + categories::add('Boxes', 'ignored'); + categories::add('Plain'); + $this->assertSame(array('Boxes' => 'B key', 'Plain' => ''), categories::found_keys()); + $this->assertSame(array('Boxes', 'Plain'), categories::found()); + categories::reset(); + } } diff --git a/tests/unit/markup_test.php b/tests/unit/markup_test.php index 8281fff..5034651 100644 --- a/tests/unit/markup_test.php +++ b/tests/unit/markup_test.php @@ -226,10 +226,19 @@ public function test_a_category_mark_becomes_a_tag_with_a_normalised_name() $this->assertSame(3, substr_count($xml, 'assertStringContainsString('', $xml); - $this->assertSame(2, substr_count($xml, ''), $xml); + $this->assertStringContainsString('', $xml, 'the sort key is kept'); + $this->assertStringContainsString('', $xml); $this->assertStringNotContainsString('parse('See [[:Category:Boxes]] and [[:Category:Boxes|all boxes]].'); + + $this->assertStringNotContainsString('assertSame(2, substr_count($xml, 'assertStringNotContainsString('parse('[[Category:Boxes]]', false)); @@ -239,6 +248,7 @@ public function test_a_category_mark_stays_text_outside_a_wiki_article_and_when_ public function test_a_category_mark_renders_as_an_empty_marker() { $this->assertSame('a b', $this->render('a [[Category:boxes]] b')); + $this->assertSame('', $this->render('[[Category:boxes|Small]]'), 'a sort key goes with it'); } public function test_template_calls_become_tags_with_their_name_and_call() diff --git a/tests/wiki/category_members_test.php b/tests/wiki/category_members_test.php new file mode 100644 index 0000000..eae0347 --- /dev/null +++ b/tests/wiki/category_members_test.php @@ -0,0 +1,117 @@ +createXMLDataSet(__DIR__ . '/fixtures/category_members.xml'); + } + + protected function setUp(): void + { + parent::setUp(); + $this->db = $this->new_dbal(); + $this->config = new \phpbb\config\config(array('wiki_category_members_built' => 0, 'wiki_category_members_cursor' => '')); + } + + /** + * @param array $answers Article names with the categories (and keys) compute() gives them + * @return category_members + */ + protected function members(array $answers) + { + global $table_prefix; + + $templates = new \phpbbmodders\wiki\wiki\templates($this->config, $this->db, $this->createMock(\phpbb\language\language::class), $table_prefix . 'wiki_article'); + $members = new class($this->config, $this->db, $this->createMock(\phpbbmodders\wiki\wiki\links::class), $templates, $table_prefix . 'wiki_article', $table_prefix . 'wiki_category_member') extends category_members { + /** @var array */ + public $answers = array(); + + public function compute($article) + { + return $this->answers[$article] ?? array(); + } + }; + $members->answers = $answers; + + return $members; + } + + public function test_members_sort_by_key_or_title_with_sub_categories_apart() + { + $members = $this->members(array( + 'Zebra' => array('Fruit' => 'Aardvark'), + 'Apple' => array('Fruit' => ''), + 'Mango' => array('Fruit' => '', 'Tropical' => ''), + 'Category:Tropical' => array('Fruit' => ''), + )); + foreach (array('Zebra', 'Apple', 'Mango', 'Category:Tropical') as $article) + { + $members->update($article); + } + + $lists = $members->members('fruit'); + $this->assertSame(array('Zebra', 'Apple', 'Mango'), array_column($lists['articles'], 'article_url'), 'Zebra is filed under its key'); + $this->assertSame(array('Category:Tropical'), array_column($lists['subcategories'], 'article_url')); + $this->assertSame(array('Fruit' => 4, 'Tropical' => 1), $members->in_use()); + } + + public function test_an_article_that_is_not_live_is_not_listed() + { + $members = $this->members(array('Draft' => array('Fruit' => ''))); + $members->update('Draft'); + + $this->assertSame(array(), $members->members('Fruit')['articles']); + } + + public function test_removing_an_article_takes_it_out_of_its_categories() + { + $members = $this->members(array('Apple' => array('Fruit' => ''))); + $members->update('Apple'); + $members->remove('Apple'); + + $this->assertSame(array(), $members->in_use()); + } + + public function test_the_cron_fills_the_table_in_batches_then_marks_it_built() + { + $members = $this->members(array('Apple' => array('Fruit' => ''), 'Zebra' => array('Fruit' => ''))); + $this->assertFalse($members->built()); + + $this->assertSame(2, $members->build_batch(2)); + $this->assertFalse($members->built(), 'a full batch means there may be more'); + $this->assertSame(2, $members->build_batch(2)); + $this->assertSame(1, $members->build_batch(2)); + $this->assertTrue($members->built()); + $this->assertSame('', (string) $this->config['wiki_category_members_cursor']); + $this->assertSame(array('Fruit' => 2), $members->in_use()); + } +} diff --git a/tests/wiki/fixtures/category_members.xml b/tests/wiki/fixtures/category_members.xml new file mode 100644 index 0000000..debac43 --- /dev/null +++ b/tests/wiki/fixtures/category_members.xml @@ -0,0 +1,26 @@ + + + + article_url + category + sort_key +
+ + article_id + article_url + article_title + article_text + article_approved + article_last_edit + article_description + article_sources + article_redirect + article_edit_reason + article_toc + 1ZebraZebra1100 + 2AppleApple1100 + 3MangoMango1100 + 4Category:TropicalCategory:Tropical1100Warm fruit + 5DraftDraft0100 +
+ diff --git a/wiki/categories.php b/wiki/categories.php index 342f077..f28904b 100644 --- a/wiki/categories.php +++ b/wiki/categories.php @@ -16,8 +16,9 @@ * are collected while an article is shown, to list under it, and the members of a * category are the live articles whose stored text holds the tag. * - * A category added by a template shows under the article that uses the template, but - * the member list of the category only knows what is written in the articles themselves. + * The member lists come from category_members once its table is built, which knows the + * categories templates add, sub-categories and sort keys; until then they are read from + * what is written in the articles themselves. */ class categories { @@ -27,6 +28,9 @@ class categories /** @var array Names found in the text rendered since reset(), in order */ protected static $found = array(); + /** @var array The sort key each of those was written with, '' for none */ + protected static $keys = array(); + /** @var \phpbb\db\driver\driver_interface */ protected $db; @@ -36,13 +40,18 @@ class categories /** @var string */ protected $article_table; + /** @var category_members */ + protected $category_members; + /** * @param \phpbb\db\driver\driver_interface $db Database object * @param \phpbbmodders\wiki\wiki\templates $templates Wiki templates service, for the live-version query * @param string $article_table Wiki article table + * @param category_members $category_members Stored memberships */ - public function __construct(\phpbb\db\driver\driver_interface $db, \phpbbmodders\wiki\wiki\templates $templates, $article_table) + public function __construct(\phpbb\db\driver\driver_interface $db, \phpbbmodders\wiki\wiki\templates $templates, $article_table, category_members $category_members) { + $this->category_members = $category_members; $this->db = $db; $this->templates = $templates; $this->article_table = $article_table; @@ -71,23 +80,37 @@ public static function normalise($name) public static function reset() { self::$found = array(); + self::$keys = array(); } /** - * Note that rendered text puts the article in a category. + * Note that rendered text puts the article in a category. The first sort key written for + * a category counts, as on MediaWiki. * * @param string $name The category's name + * @param string $key The sort key after the bar, '' for none * @return void */ - public static function add($name) + public static function add($name, $key = '') { $name = self::normalise($name); if ($name !== '' && !in_array($name, self::$found, true)) { self::$found[] = $name; + self::$keys[$name] = trim((string) $key); } } + /** + * The sort keys of the categories found since reset(): names with their keys. + * + * @return array + */ + public static function found_keys() + { + return self::$keys; + } + /** * The categories found in the text rendered since reset(), in the order they were written. * @@ -146,20 +169,31 @@ protected function members_where($category) * @return array Normalised names, unique, in order */ public static function names_outside_includeonly($raw) + { + return array_keys(self::keys_outside_includeonly($raw)); + } + + /** + * The categories a template page puts itself in, with their sort keys. + * + * @param string $raw The template page's text as it is written + * @return array Normalised names, in order, with the first sort key written for each ('' for none) + */ + public static function keys_outside_includeonly($raw) { $raw = preg_replace('#.*?#is', '', $raw); - if (!preg_match_all('/\[\[\s*Category:\s*([^\[\]|\n]+?)\s*(?:\|[^\[\]\n]*)?\]\]/i', $raw, $matches)) + if (!preg_match_all('/\[\[\s*Category:\s*([^\[\]|\n]+?)\s*(?:\|\s*([^\[\]\n]*?)\s*)?\]\]/i', $raw, $matches, PREG_SET_ORDER)) { return array(); } $names = array(); - foreach ($matches[1] as $name) + foreach ($matches as $match) { - $name = self::normalise($name); - if ($name !== '' && !in_array($name, $names, true)) + $name = self::normalise($match[1]); + if ($name !== '' && !isset($names[$name])) { - $names[] = $name; + $names[$name] = isset($match[2]) ? trim($match[2]) : ''; } } @@ -186,24 +220,30 @@ protected function names_of($row) } /** - * Whether any live article is in a category. + * Whether any live article, or sub-category, is in a category. * * @param string $category * @return bool */ public function has_members($category) { - return count($this->members($category)) > 0; + return count($this->members($category)) > 0 || count($this->subcategories($category)) > 0; } /** - * Every live article in a category, A to Z. + * Every live article in a category, by sort key or else title (A to Z before the stored + * memberships are built), sub-categories left out. * * @param string $category * @return array Rows with article_url, article_title and article_description */ public function members($category) { + if ($this->category_members->built()) + { + return $this->category_members->members($category)['articles']; + } + $normal = self::normalise($category); $sql = 'SELECT a.article_url, a.article_title, a.article_description, a.article_text, a.bbcode_uid FROM ' . $this->article_table . ' a @@ -226,6 +266,18 @@ public function members($category) return $members; } + /** + * The sub-categories of a category: category pages that are themselves in it, by sort + * key or else title. Known once the stored memberships are built. + * + * @param string $category + * @return array Rows with article_url, article_title and article_description + */ + public function subcategories($category) + { + return $this->category_members->built() ? $this->category_members->members($category)['subcategories'] : array(); + } + /** * The short description of every category that has a page: the description field of * the live version of its "Category:Name" article. @@ -255,6 +307,11 @@ public function descriptions() */ public function in_use() { + if ($this->category_members->built()) + { + return $this->category_members->in_use(); + } + $sql = 'SELECT a.article_url, a.article_text, a.bbcode_uid FROM ' . $this->article_table . ' a WHERE ' . $this->templates->live_where('a', '') . ' diff --git a/wiki/category_members.php b/wiki/category_members.php new file mode 100644 index 0000000..83b3aa4 --- /dev/null +++ b/wiki/category_members.php @@ -0,0 +1,301 @@ + parts. They are worked out again whenever the live version + * changes, and for every article that uses a template when the template's live version + * changes. A cron task fills the table for the articles a board already has (see built()). + */ +class category_members +{ + /** How many articles one cron run works out */ + const BATCH = 50; + + /** @var \phpbb\config\config */ + protected $config; + + /** @var \phpbb\db\driver\driver_interface */ + protected $db; + + /** @var links */ + protected $links; + + /** @var templates */ + protected $templates; + + /** @var string */ + protected $article_table; + + /** @var string */ + protected $member_table; + + /** + * @param \phpbb\config\config $config + * @param \phpbb\db\driver\driver_interface $db + * @param links $links + * @param templates $templates + * @param string $article_table + * @param string $member_table + */ + public function __construct(\phpbb\config\config $config, \phpbb\db\driver\driver_interface $db, links $links, templates $templates, $article_table, $member_table) + { + $this->config = $config; + $this->db = $db; + $this->links = $links; + $this->templates = $templates; + $this->article_table = $article_table; + $this->member_table = $member_table; + } + + /** + * Whether the table holds every article yet. Until it does, the category lists read the + * articles' text as before. + * + * @return bool + */ + public function built() + { + return !empty($this->config['wiki_category_members_built']); + } + + /** + * The categories an article's live version puts it in, with their sort keys. + * + * @param string $article + * @return array Normalised category names with their sort keys ('' for none) + */ + public function compute($article) + { + $sql = 'SELECT a.article_url, a.article_text, a.bbcode_uid, a.bbcode_bitfield + FROM ' . $this->article_table . ' a + WHERE ' . $this->templates->live_where('a', '') . " + AND a.article_url = '" . $this->db->sql_escape($article) . "'"; + $result = $this->db->sql_query_limit($sql, 1); + $row = $this->db->sql_fetchrow($result); + $this->db->sql_freeresult($result); + if (!$row) + { + return array(); + } + + if (strpos($article, templates::PREFIX) === 0) + { + $edit = generate_text_for_edit($row['article_text'], $row['bbcode_uid'], OPTION_FLAG_BBCODE | OPTION_FLAG_SMILIES | OPTION_FLAG_LINKS); + + return categories::keys_outside_includeonly(htmlspecialchars_decode($edit['text'], ENT_COMPAT)); + } + + // Rendering notes every category mark, the templates' too (see the wikilink listener) + $was_found = categories::found_keys(); + categories::reset(); + files::set_context($article); + try + { + generate_text_for_display($row['article_text'], $row['bbcode_uid'], $row['bbcode_bitfield'], OPTION_FLAG_BBCODE | OPTION_FLAG_SMILIES | OPTION_FLAG_LINKS); + $keys = categories::found_keys(); + } + finally + { + files::set_context(null); + categories::reset(); + foreach ($was_found as $name => $key) + { + categories::add($name, $key); + } + } + + return $keys; + } + + /** + * Work out the article's categories again, and when it is a template, those of every + * article that uses it (through other templates too). + * + * @param string $article + * @param array $done Articles already worked out in this round, so template loops end + * @return void + */ + public function update($article, array &$done = array()) + { + if (isset($done[$article])) + { + return; + } + $done[$article] = true; + + $this->store($article, $this->compute($article)); + + if (strpos($article, templates::PREFIX) === 0) + { + foreach ($this->links->what_links_here($article) as $user) + { + $this->update($user['article_url'], $done); + } + } + } + + /** + * The article is gone: it is in no category any more, and the articles that used it, if + * it was a template, lose what it gave them. + * + * @param string $article + * @param array $users For a template, the articles that used it, read before it went + * @return void + */ + public function remove($article, array $users = array()) + { + $this->store($article, array()); + $done = array($article => true); + foreach ($users as $user) + { + $this->update($user, $done); + } + } + + /** + * Replace the stored categories of an article. + * + * @param string $article + * @param array $keys Category names with their sort keys + * @return void + */ + protected function store($article, array $keys) + { + $this->db->sql_transaction('begin'); + $this->db->sql_query('DELETE FROM ' . $this->member_table . " + WHERE article_url = '" . $this->db->sql_escape($article) . "'"); + foreach ($keys as $category => $key) + { + $this->db->sql_query('INSERT INTO ' . $this->member_table . ' ' . $this->db->sql_build_array('INSERT', array( + 'article_url' => $article, + 'category' => utf8_substr((string) $category, 0, 255), + 'sort_key' => utf8_substr((string) $key, 0, 255), + ))); + } + $this->db->sql_transaction('commit'); + } + + /** + * One cron run of filling the table for the articles the board already had: the next + * articles by name after where the last run stopped. When none are left, the table is + * marked built and the category lists start reading it. + * + * @param int $limit + * @return int How many articles this run worked out + */ + public function build_batch($limit = self::BATCH) + { + $after = (string) $this->config['wiki_category_members_cursor']; + $sql = 'SELECT DISTINCT article_url + FROM ' . $this->article_table . " + WHERE article_url > '" . $this->db->sql_escape($after) . "' + ORDER BY article_url ASC"; + $result = $this->db->sql_query_limit($sql, (int) $limit); + $articles = array(); + while ($row = $this->db->sql_fetchrow($result)) + { + $articles[] = $row['article_url']; + } + $this->db->sql_freeresult($result); + + foreach ($articles as $article) + { + $this->store($article, $this->compute($article)); + } + + if (count($articles) < $limit) + { + $this->config->set('wiki_category_members_built', 1); + $this->config->set('wiki_category_members_cursor', ''); + } + else + { + $this->config->set('wiki_category_members_cursor', end($articles)); + } + + return count($articles); + } + + /** + * The live articles in a category, with their sort keys: sub-categories (category pages) + * and the other articles apart, each sorted by key, or by title where there is none. + * + * @param string $category + * @return array Two lists, 'subcategories' and 'articles', of rows with article_url, + * article_title, article_description and sort_key + */ + public function members($category) + { + $sql = 'SELECT a.article_url, a.article_title, a.article_description, m.sort_key + FROM ' . $this->member_table . ' m + JOIN ' . $this->article_table . ' a ON a.article_url = m.article_url + WHERE m.category = \'' . $this->db->sql_escape(categories::normalise($category)) . '\' + AND ' . $this->templates->live_where('a', ''); + $result = $this->db->sql_query($sql); + $lists = array('subcategories' => array(), 'articles' => array()); + while ($row = $this->db->sql_fetchrow($result)) + { + $lists[strpos($row['article_url'], categories::PREFIX) === 0 ? 'subcategories' : 'articles'][] = $row; + } + $this->db->sql_freeresult($result); + + foreach ($lists as &$list) + { + usort($list, array(self::class, 'compare')); + } + + return $lists; + } + + /** + * Order two members: by sort key, or title where there is none, ignoring case. + * + * @param array $a + * @param array $b + * @return int + */ + public static function compare(array $a, array $b) + { + $key_a = ($a['sort_key'] !== '') ? $a['sort_key'] : htmlspecialchars_decode($a['article_title'], ENT_COMPAT); + $key_b = ($b['sort_key'] !== '') ? $b['sort_key'] : htmlspecialchars_decode($b['article_title'], ENT_COMPAT); + + return strnatcasecmp($key_a, $key_b) ?: strcmp($a['article_url'], $b['article_url']); + } + + /** + * Every category with a live article in it, and how many (sub-categories included). + * + * @return array Category names with their member counts, A to Z + */ + public function in_use() + { + $sql = 'SELECT m.category, COUNT(m.article_url) AS total + FROM ' . $this->member_table . ' m + JOIN ' . $this->article_table . ' a ON a.article_url = m.article_url + WHERE ' . $this->templates->live_where('a', '') . ' + GROUP BY m.category'; + $result = $this->db->sql_query($sql); + $counts = array(); + while ($row = $this->db->sql_fetchrow($result)) + { + $counts[$row['category']] = (int) $row['total']; + } + $this->db->sql_freeresult($result); + ksort($counts, SORT_STRING | SORT_FLAG_CASE); + + return $counts; + } +} diff --git a/wiki/delete.php b/wiki/delete.php index 37e2d8f..fc66a34 100644 --- a/wiki/delete.php +++ b/wiki/delete.php @@ -13,6 +13,9 @@ class delete extends \phpbbmodders\wiki\wiki\functions { + /** @var \phpbbmodders\wiki\wiki\links */ + protected $links; + /** @var \phpbbmodders\wiki\wiki\protection */ protected $protection; @@ -56,8 +59,10 @@ class delete extends \phpbbmodders\wiki\wiki\functions * @param \phpbb\request\request $request Request object * @param \phpbbmodders\wiki\wiki\watch $watch Watched articles */ - public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\user $user, $article_table, \phpbbmodders\wiki\wiki\files $files, \phpbb\request\request $request, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\search_sync $search_sync, \phpbbmodders\wiki\wiki\protection $protection) + public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\user $user, $article_table, \phpbbmodders\wiki\wiki\files $files, \phpbb\request\request $request, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\search_sync $search_sync, \phpbbmodders\wiki\wiki\protection $protection, \phpbbmodders\wiki\wiki\category_members $category_members, \phpbbmodders\wiki\wiki\links $links) { + $this->category_members = $category_members; + $this->links = $links; $this->protection = $protection; $this->search_sync = $search_sync; $this->files = $files; @@ -131,6 +136,8 @@ public function article($article) // Out of the search first, while its versions can still be found $this->search_sync->remove($article); + // A template's users, read while its versions still exist, lose what it gave them + $template_users = (strpos($article, \phpbbmodders\wiki\wiki\templates::PREFIX) === 0) ? array_column($this->links->what_links_here($article), 'article_url') : array(); $sql = 'DELETE FROM ' . $this->article_table . " WHERE article_url = '" . $this->db->sql_escape($article) . "'"; @@ -138,6 +145,7 @@ public function article($article) $this->files->delete_all($article); $this->watch->delete_for_article($article); $this->protection->delete_for_article($article); + $this->category_members->remove($article, $template_users); trigger_error($this->user->lang('DELETE_ARTICLE_SUCCESS') . '

' . $this->user->lang('BACK_TO_WIKI') . ''); } else diff --git a/wiki/edit.php b/wiki/edit.php index 3fba486..57abc21 100644 --- a/wiki/edit.php +++ b/wiki/edit.php @@ -87,9 +87,11 @@ class edit extends \phpbbmodders\wiki\wiki\functions * @param \phpbbmodders\wiki\wiki\search_sync $search_sync Keeps the search current * @param \phpbbmodders\wiki\wiki\edit_lock $edit_lock Who has an article's edit form open * @param \phpbbmodders\wiki\wiki\protection $protection Which groups may edit an article + * @param \phpbbmodders\wiki\wiki\category_members $category_members Which articles are in which category */ - public function __construct(\phpbb\auth\auth $auth, \phpbb\config\config $config, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\notification\manager $notification_manager, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, $article_table, $phpbb_root_path, $php_ext, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\templates $templates, \phpbbmodders\wiki\wiki\search_sync $search_sync, \phpbbmodders\wiki\wiki\edit_lock $edit_lock, \phpbbmodders\wiki\wiki\protection $protection) + public function __construct(\phpbb\auth\auth $auth, \phpbb\config\config $config, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\notification\manager $notification_manager, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, $article_table, $phpbb_root_path, $php_ext, \phpbbmodders\wiki\wiki\watch $watch, \phpbbmodders\wiki\wiki\templates $templates, \phpbbmodders\wiki\wiki\search_sync $search_sync, \phpbbmodders\wiki\wiki\edit_lock $edit_lock, \phpbbmodders\wiki\wiki\protection $protection, \phpbbmodders\wiki\wiki\category_members $category_members) { + $this->category_members = $category_members; $this->protection = $protection; $this->edit_lock = $edit_lock; $this->search_sync = $search_sync; diff --git a/wiki/functions.php b/wiki/functions.php index 352bbbe..35efcaf 100644 --- a/wiki/functions.php +++ b/wiki/functions.php @@ -15,8 +15,11 @@ class functions /** @var \phpbbmodders\wiki\wiki\search_sync|null Keeps the search current; set by the classes that change articles */ protected $search_sync = null; + /** @var \phpbbmodders\wiki\wiki\category_members|null Keeps the category lists current; set with search_sync */ + protected $category_members = null; + /** - * Tell the search that the live version of an article may have changed. + * Tell the search and the category lists that the live version of an article may have changed. * * @param string $article_url * @return void @@ -27,6 +30,10 @@ protected function sync_search($article_url) { $this->search_sync->sync($article_url); } + if ($this->category_members !== null) + { + $this->category_members->update($article_url); + } } /** @var \phpbb\auth\auth */ diff --git a/wiki/markup.php b/wiki/markup.php index c930f09..7eba6f9 100644 --- a/wiki/markup.php +++ b/wiki/markup.php @@ -85,6 +85,18 @@ public static function decode_entities_in_name($value) return str_replace("\u{00A0}", ' ', self::decode_entities($value)); } + /** + * A link written [[:Category:Name]] (or [[:File:...]]) goes to that page: the colon only + * keeps the link from being read as a category mark, so it is taken off the name. + * + * @param string $value The link's target as written + * @return string + */ + public static function strip_link_colon($value) + { + return (strpos($value, ':') === 0) ? ltrim(substr($value, 1)) : $value; + } + /** * Switch wiki markup on for the next parse (an article save or preview), or off. * diff --git a/wiki/move.php b/wiki/move.php index edc7ce1..83941b0 100644 --- a/wiki/move.php +++ b/wiki/move.php @@ -52,6 +52,9 @@ class move /** @var protection */ protected $protection; + /** @var category_members */ + protected $category_members; + /** @var string */ protected $watch_table; @@ -68,9 +71,11 @@ class move * @param string $file_table * @param string $watch_table * @param protection $protection + * @param category_members $category_members */ - public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, links $links, search_sync $search_sync, $article_table, $file_table, $watch_table, protection $protection) + public function __construct(\phpbb\auth\auth $auth, \phpbb\db\driver\driver_interface $db, \phpbb\controller\helper $helper, \phpbb\request\request $request, \phpbb\template\template $template, \phpbb\user $user, links $links, search_sync $search_sync, $article_table, $file_table, $watch_table, protection $protection, category_members $category_members) { + $this->category_members = $category_members; $this->protection = $protection; $this->auth = $auth; $this->db = $db; @@ -310,5 +315,9 @@ protected function move_article($old, $new, $title, $leave_redirect, $old_title) // and the redirect left behind are brought up to date $this->search_sync->sync($new); $this->search_sync->sync($old); + // The article's categories go with it; a template's users are worked out again, as + // they now reach it through the redirect + $this->category_members->update($new); + $this->category_members->update($old); } }