Index: trunk/web/punbb/include/search_idx.php
===================================================================
--- trunk/web/punbb/include/search_idx.php	(revision 10)
+++ trunk/web/punbb/include/search_idx.php	(revision 1)
@@ -1,12 +1,29 @@
 <?php
-
-/**
- * Copyright (C) 2008-2011 FluxBB
- * based on code by Rickard Andersson copyright (C) 2002-2008 PunBB
- * License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher
- */
+/***********************************************************************
+
+  Copyright (C) 2002-2005  Rickard Andersson (rickard@punbb.org)
+
+  This file is part of PunBB.
+
+  PunBB is free software; you can redistribute it and/or modify it
+  under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 2 of the License,
+  or (at your option) any later version.
+
+  PunBB is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+  MA  02111-1307  USA
+
+************************************************************************/
+
 
 // The contents of this file are very much inspired by the file functions_search.php
-// from the phpBB Group forum software phpBB2 (http://www.phpbb.com)
+// from the phpBB Group forum software phpBB2 (http://www.phpbb.com). 
 
 
@@ -16,138 +33,50 @@
 
 
-// Make a regex that will match CJK or Hangul characters
-define('PUN_CJK_HANGUL_REGEX', '['.
-	'\x{1100}-\x{11FF}'.		// Hangul Jamo							1100-11FF		(http://www.fileformat.info/info/unicode/block/hangul_jamo/index.htm)
-	'\x{3130}-\x{318F}'.		// Hangul Compatibility Jamo			3130-318F		(http://www.fileformat.info/info/unicode/block/hangul_compatibility_jamo/index.htm)
-	'\x{AC00}-\x{D7AF}'.		// Hangul Syllables						AC00-D7AF		(http://www.fileformat.info/info/unicode/block/hangul_syllables/index.htm)
-
-	// Hiragana
-	'\x{3040}-\x{309F}'.		// Hiragana								3040-309F		(http://www.fileformat.info/info/unicode/block/hiragana/index.htm)
-
-	// Katakana
-	'\x{30A0}-\x{30FF}'.		// Katakana								30A0-30FF		(http://www.fileformat.info/info/unicode/block/katakana/index.htm)
-	'\x{31F0}-\x{31FF}'.		// Katakana Phonetic Extensions			31F0-31FF		(http://www.fileformat.info/info/unicode/block/katakana_phonetic_extensions/index.htm)
-
-	// CJK Unified Ideographs	(http://en.wikipedia.org/wiki/CJK_Unified_Ideographs)
-	'\x{2E80}-\x{2EFF}'.		// CJK Radicals Supplement				2E80-2EFF		(http://www.fileformat.info/info/unicode/block/cjk_radicals_supplement/index.htm)
-	'\x{2F00}-\x{2FDF}'.		// Kangxi Radicals						2F00-2FDF		(http://www.fileformat.info/info/unicode/block/kangxi_radicals/index.htm)
-	'\x{2FF0}-\x{2FFF}'.		// Ideographic Description Characters	2FF0-2FFF		(http://www.fileformat.info/info/unicode/block/ideographic_description_characters/index.htm)
-	'\x{3000}-\x{303F}'.		// CJK Symbols and Punctuation			3000-303F		(http://www.fileformat.info/info/unicode/block/cjk_symbols_and_punctuation/index.htm)
-	'\x{31C0}-\x{31EF}'.		// CJK Strokes							31C0-31EF		(http://www.fileformat.info/info/unicode/block/cjk_strokes/index.htm)
-	'\x{3200}-\x{32FF}'.		// Enclosed CJK Letters and Months		3200-32FF		(http://www.fileformat.info/info/unicode/block/enclosed_cjk_letters_and_months/index.htm)
-	'\x{3400}-\x{4DBF}'.		// CJK Unified Ideographs Extension A	3400-4DBF		(http://www.fileformat.info/info/unicode/block/cjk_unified_ideographs_extension_a/index.htm)
-	'\x{4E00}-\x{9FFF}'.		// CJK Unified Ideographs				4E00-9FFF		(http://www.fileformat.info/info/unicode/block/cjk_unified_ideographs/index.htm)
-	'\x{20000}-\x{2A6DF}'.		// CJK Unified Ideographs Extension B	20000-2A6DF		(http://www.fileformat.info/info/unicode/block/cjk_unified_ideographs_extension_b/index.htm)
-']');
-
-
 //
 // "Cleans up" a text string and returns an array of unique words
 // This function depends on the current locale setting
 //
-function split_words($text, $idx)
+function split_words($text)
 {
-	// Remove BBCode
-	$text = preg_replace('%\[/?(b|u|s|ins|del|em|i|h|colou?r|quote|code|img|url|email|list)(?:\=[^\]]*)?\]%', ' ', $text);
-
-	// Remove any apostrophes or dashes which aren't part of words
-	$text = substr(ucp_preg_replace('%((?<=[^\p{L}\p{N}])[\'\-]|[\'\-](?=[^\p{L}\p{N}]))%u', '', ' '.$text.' '), 1, -1);
-
-	// Remove punctuation and symbols (actually anything that isn't a letter or number), allow apostrophes and dashes (and % * if we aren't indexing)
-	$text = ucp_preg_replace('%(?![\'\-'.($idx ? '' : '\%\*').'])[^\p{L}\p{N}]+%u', ' ', $text);
-
-	// Replace multiple whitespace or dashes
-	$text = preg_replace('%(\s){2,}%u', '\1', $text);
+	global $pun_user;
+	static $noise_match, $noise_replace, $stopwords;
+
+	if (empty($noise_match))
+	{
+		$noise_match = array('[quote', '[code', '[url', '[img', '[email', '[color', '[colour', 'quote]', 'code]', 'url]', 'img]', 'email]', 'color]', 'colour]', '^', '$', '&', '(', ')', '<', '>', '`', '\'', '"', '|', ',', '@', '_', '?', '%', '~', '+', '[', ']', '{', '}', ':', '\\', '/', '=', '#', ';', '!', '*');
+		$noise_replace = array('',       '',      '',     '',     '',       '',       '',        '',       '',      '',     '',     '',       '',       '',        ' ', ' ', ' ', ' ', ' ', ' ', ' ', '',  '',   ' ', ' ', ' ', ' ', '',  ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '' ,  ' ', ' ', ' ', ' ', ' ', ' ');
+
+		$stopwords = (array)@file(PUN_ROOT.'lang/'.$pun_user['language'].'/stopwords.txt');
+		$stopwords = array_map('trim', $stopwords);
+	}
+
+	// Clean up
+	$patterns[] = '#&[\#a-z0-9]+?;#i';
+	$patterns[] = '#\b[\w]+:\/\/[a-z0-9\.\-]+(\/[a-z0-9\?\.%_\-\+=&\/~]+)?#';
+	$patterns[] = '#\[\/?[a-z\*=\+\-]+(\:?[0-9a-z]+)?:[a-z0-9]{10,}(\:[a-z0-9]+)?=?.*?\]#';
+	$text = preg_replace($patterns, ' ', ' '.strtolower($text).' ');
+
+	// Filter out junk
+	$text = str_replace($noise_match, $noise_replace, $text);
+
+	// Strip out extra whitespace between words
+	$text = trim(preg_replace('#\s+#', ' ', $text));
 
 	// Fill an array with all the words
-	$words = array_unique(explode(' ', $text));
-
-	// Remove any words that should not be indexed
-	foreach ($words as $key => $value)
-	{
-		// If the word shouldn't be indexed, remove it
-		if (!validate_search_word($value, $idx))
-			unset($words[$key]);
-	}
-
-	return $words;
-}
-
-
-//
-// Checks if a word is a valid searchable word
-//
-function validate_search_word($word, $idx)
-{
-	static $stopwords;
-
-	// If the word is a keyword we don't want to index it, but we do want to be allowed to search it
-	if (is_keyword($word))
-		return !$idx;
-
-	if (!isset($stopwords))
-	{
-		if (file_exists(FORUM_CACHE_DIR.'cache_stopwords.php'))
-			include FORUM_CACHE_DIR.'cache_stopwords.php';
-
-		if (!defined('PUN_STOPWORDS_LOADED'))
-		{
-			if (!defined('FORUM_CACHE_FUNCTIONS_LOADED'))
-				require PUN_ROOT.'include/cache.php';
-
-			generate_stopwords_cache();
-			require FORUM_CACHE_DIR.'cache_stopwords.php';
-		}
-	}
-
-	// If it is a stopword it isn't valid
-	if (in_array($word, $stopwords))
-		return false;
-
-	// If the word if CJK we don't want to index it, but we do want to be allowed to search it
-	if (is_cjk($word))
-		return !$idx;
-
-	// Check the word is within the min/max length
-	$num_chars = pun_strlen($word);
-	return $num_chars >= PUN_SEARCH_MIN_WORD && $num_chars <= PUN_SEARCH_MAX_WORD;
-}
-
-
-//
-// Check a given word is a search keyword.
-//
-function is_keyword($word)
-{
-	return $word == 'and' || $word == 'or' || $word == 'not';
-}
-
-
-//
-// Check if a given word is CJK or Hangul.
-//
-function is_cjk($word)
-{
-	return preg_match('%^'.PUN_CJK_HANGUL_REGEX.'+$%u', $word) ? true : false;
-}
-
-
-//
-// Strip [img] [url] and [email] out of the message so we don't index their contents
-//
-function strip_bbcode($text)
-{
-	static $patterns;
-
-	if (!isset($patterns))
-	{
-		$patterns = array(
-			'%\[img=([^\]]*+)\]([^[]*+)\[/img\]%'									=>	'$2 $1',	// Keep the url and description
-			'%\[(url|email)=([^\]]*+)\]([^[]*+(?:(?!\[/\1\])\[[^[]*+)*)\[/\1\]%'	=>	'$2 $3',	// Keep the url and text
-			'%\[(img|url|email)\]([^[]*+(?:(?!\[/\1\])\[[^[]*+)*)\[/\1\]%'			=>	'$2',		// Keep the url
-		);
-	}
-
-	return preg_replace(array_keys($patterns), array_values($patterns), $text);
+	$words = explode(' ', $text);
+
+	if (!empty($words))
+	{
+		while (list($i, $word) = @each($words))
+		{
+			$words[$i] = trim($word, '.');
+			$num_chars = pun_strlen($word);
+
+			if ($num_chars < 3 || $num_chars > 20 || in_array($word, $stopwords))
+				unset($words[$i]);
+		}
+	}
+
+	return array_unique($words);
 }
 
@@ -160,17 +89,11 @@
 	global $db_type, $db;
 
-	$message = utf8_strtolower($message);
-	$subject = utf8_strtolower($subject);
-
-	// Remove any bbcode that we shouldn't index
-	$message = strip_bbcode($message);
-
 	// Split old and new post/subject to obtain array of 'words'
-	$words_message = split_words($message, true);
-	$words_subject = ($subject) ? split_words($subject, true) : array();
+	$words_message = split_words($message);
+	$words_subject = ($subject) ? split_words($subject) : array();
 
 	if ($mode == 'edit')
 	{
-		$result = $db->query('SELECT w.id, w.word, m.subject_match FROM '.$db->prefix.'search_words AS w INNER JOIN '.$db->prefix.'search_matches AS m ON w.id=m.word_id WHERE m.post_id='.$post_id, true) or error('Unable to fetch search index words', __FILE__, __LINE__, $db->error());
+		$result = $db->query('SELECT w.id, w.word, m.subject_match FROM '.$db->prefix.'search_words AS w INNER JOIN '.$db->prefix.'search_matches AS m ON w.id=m.word_id WHERE m.post_id='.$post_id, true) or error('Impossible de retrouver les mots index de recherches', __FILE__, __LINE__, $db->error());
 
 		// Declare here to stop array_keys() and array_diff() from complaining if not set
@@ -207,5 +130,5 @@
 	if (!empty($unique_words))
 	{
-		$result = $db->query('SELECT id, word FROM '.$db->prefix.'search_words WHERE word IN(\''.implode('\',\'', array_map(array($db, 'escape'), $unique_words)).'\')', true) or error('Unable to fetch search index words', __FILE__, __LINE__, $db->error());
+		$result = $db->query('SELECT id, word FROM '.$db->prefix.'search_words WHERE word IN('.implode(',', preg_replace('#^(.*)$#', '\'\1\'', $unique_words)).')', true) or error('Impossible de retrouver les mots index de recherches', __FILE__, __LINE__, $db->error());
 
 		$word_ids = array();
@@ -224,12 +147,10 @@
 				case 'mysql':
 				case 'mysqli':
-				case 'mysql_innodb':
-				case 'mysqli_innodb':
-					$db->query('INSERT INTO '.$db->prefix.'search_words (word) VALUES(\''.implode('\'),(\'', array_map(array($db, 'escape'), $new_words)).'\')');
+					$db->query('INSERT INTO '.$db->prefix.'search_words (word) VALUES'.implode(',', preg_replace('#^(.*)$#', '(\'\1\')', $new_words))) or error('Impossible d\'ajouter les mots index de recherche', __FILE__, __LINE__, $db->error());
 					break;
 
 				default:
-					foreach ($new_words as $word)
-						$db->query('INSERT INTO '.$db->prefix.'search_words (word) VALUES(\''.$db->escape($word).'\')');
+					while (list(, $word) = @each($new_words))
+						$db->query('INSERT INTO '.$db->prefix.'search_words (word) VALUES(\''.$word.'\')') or error('Impossible d\'ajouter les mots index de recherche', __FILE__, __LINE__, $db->error());
 					break;
 			}
@@ -240,5 +161,5 @@
 
 	// Delete matches (only if editing a post)
-	foreach ($words['del'] as $match_in => $wordlist)
+	while (list($match_in, $wordlist) = @each($words['del']))
 	{
 		$subject_match = ($match_in == 'subject') ? 1 : 0;
@@ -247,18 +168,18 @@
 		{
 			$sql = '';
-			foreach ($wordlist as $word)
+			while (list(, $word) = @each($wordlist))
 				$sql .= (($sql != '') ? ',' : '').$cur_words[$match_in][$word];
 
-			$db->query('DELETE FROM '.$db->prefix.'search_matches WHERE word_id IN('.$sql.') AND post_id='.$post_id.' AND subject_match='.$subject_match) or error('Unable to delete search index word matches', __FILE__, __LINE__, $db->error());
+			$db->query('DELETE FROM '.$db->prefix.'search_matches WHERE word_id IN('.$sql.') AND post_id='.$post_id.' AND subject_match='.$subject_match) or error('Impossible de supprimer des mots index de recherche', __FILE__, __LINE__, $db->error());
 		}
 	}
 
 	// Add new matches
-	foreach ($words['add'] as $match_in => $wordlist)
+	while (list($match_in, $wordlist) = @each($words['add']))
 	{
 		$subject_match = ($match_in == 'subject') ? 1 : 0;
 
 		if (!empty($wordlist))
-			$db->query('INSERT INTO '.$db->prefix.'search_matches (post_id, word_id, subject_match) SELECT '.$post_id.', id, '.$subject_match.' FROM '.$db->prefix.'search_words WHERE word IN(\''.implode('\',\'', array_map(array($db, 'escape'), $wordlist)).'\')') or error('Unable to insert search index word matches', __FILE__, __LINE__, $db->error());
+			$db->query('INSERT INTO '.$db->prefix.'search_matches (post_id, word_id, subject_match) SELECT '.$post_id.', id, '.$subject_match.' FROM '.$db->prefix.'search_words WHERE word IN('.implode(',', preg_replace('#^(.*)$#', '\'\1\'', $wordlist)).')') or error('Impossible d\'ajouter les correspondances index de recherche', __FILE__, __LINE__, $db->error());
 	}
 
@@ -278,8 +199,6 @@
 		case 'mysql':
 		case 'mysqli':
-		case 'mysql_innodb':
-		case 'mysqli_innodb':
-		{
-			$result = $db->query('SELECT word_id FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.') GROUP BY word_id') or error('Unable to fetch search index word match', __FILE__, __LINE__, $db->error());
+		{
+			$result = $db->query('SELECT word_id FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.') GROUP BY word_id') or error('Impossible de retrouver les correspondances de mots index de recherche', __FILE__, __LINE__, $db->error());
 
 			if ($db->num_rows($result))
@@ -289,5 +208,5 @@
 					$word_ids .= ($word_ids != '') ? ','.$row[0] : $row[0];
 
-				$result = $db->query('SELECT word_id FROM '.$db->prefix.'search_matches WHERE word_id IN('.$word_ids.') GROUP BY word_id HAVING COUNT(word_id)=1') or error('Unable to fetch search index word match', __FILE__, __LINE__, $db->error());
+				$result = $db->query('SELECT word_id FROM '.$db->prefix.'search_matches WHERE word_id IN('.$word_ids.') GROUP BY word_id HAVING COUNT(word_id)=1') or error('Impossible de retrouver les correspondances de mots index de recherche', __FILE__, __LINE__, $db->error());
 
 				if ($db->num_rows($result))
@@ -297,5 +216,5 @@
 						$word_ids .= ($word_ids != '') ? ','.$row[0] : $row[0];
 
-					$db->query('DELETE FROM '.$db->prefix.'search_words WHERE id IN('.$word_ids.')') or error('Unable to delete search index word', __FILE__, __LINE__, $db->error());
+					$db->query('DELETE FROM '.$db->prefix.'search_words WHERE id IN('.$word_ids.')') or error('Impossible de supprimer des mots index de recherche', __FILE__, __LINE__, $db->error());
 				}
 			}
@@ -305,8 +224,8 @@
 
 		default:
-			$db->query('DELETE FROM '.$db->prefix.'search_words WHERE id IN(SELECT word_id FROM '.$db->prefix.'search_matches WHERE word_id IN(SELECT word_id FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.') GROUP BY word_id) GROUP BY word_id HAVING COUNT(word_id)=1)') or error('Unable to delete from search index', __FILE__, __LINE__, $db->error());
+			$db->query('DELETE FROM '.$db->prefix.'search_words WHERE id IN(SELECT word_id FROM '.$db->prefix.'search_matches WHERE word_id IN(SELECT word_id FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.') GROUP BY word_id) GROUP BY word_id HAVING COUNT(word_id)=1)') or error('Impossible de supprimer depuis l\'index de recherche', __FILE__, __LINE__, $db->error());
 			break;
 	}
 
-	$db->query('DELETE FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.')') or error('Unable to delete search index word match', __FILE__, __LINE__, $db->error());
+	$db->query('DELETE FROM '.$db->prefix.'search_matches WHERE post_id IN('.$post_ids.')') or error('Impossible de supprimer des correspondances de mots index de recherche', __FILE__, __LINE__, $db->error());
 }
