rtext : initialize, replace default tokenizer

petermeissner · petermeissner · commit 1df76a5e5947 · 2016-07-13T10:41:18.000+02:00
diff --git a/R/rtext.R b/R/rtext.R
@@ -209,7 +209,7 @@ rtext <-
       function(
         text        = NULL,
         text_file   = NULL,
-        tokenizer   = rtext_tokenizer$words,
+        tokenizer   = function(x){text_tokenize(x, "\n", non_token = TRUE)},
         encoding    = "UTF-8",
         id          = NULL,
         tokenize_by = NULL,
diff --git a/R/rtext_tools.R b/R/rtext_tools.R
@@ -7,7 +7,8 @@ dp_storage <- new.env(parent = emptyenv())
 #' @export
 rtext_tokenizer <- list(
   words  = function(x){text_tokenize_words(x, non_token = TRUE )},
-  words2 = function(x){text_tokenize_words(x, non_token = FALSE)}
+  words2 = function(x){text_tokenize_words(x, non_token = FALSE)},
+  lines  = function(x){text_tokenize(x, "\n", non_token = TRUE)}
 )
 
 
diff --git a/man/rtext_tokenizer.Rd b/man/rtext_tokenizer.Rd

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,8 @@ dp_storage <- new.env(parent = emptyenv())`
`7`	`7`	`#' @export`
`8`	`8`	`rtext_tokenizer <- list(`
`9`	`9`	`words = function(x){text_tokenize_words(x, non_token = TRUE )},`
`10`		`- words2 = function(x){text_tokenize_words(x, non_token = FALSE)}`
	`10`	`+ words2 = function(x){text_tokenize_words(x, non_token = FALSE)},`
	`11`	`+ lines = function(x){text_tokenize(x, "\n", non_token = TRUE)}`
`11`	`12`	`)`
`12`	`13`
`13`	`14`