novabbs
/
rocksolid-light
3
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

Add body search feature per section

This commit is contained in:
Retro_Guy 2021-03-10 23:55:01 -07:00
parent c6400cd791
commit c06a916037
4 changed files with 134 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Rocksolid_Light/common/style.css
View File

@ -71,6 +71,10 @@
color: var(--color-visited); color: var(--color-visited);
} }
.search_result {
color: var(--color-text-alt);
}
/* Title */ /* Title */
.np_title { .np_title {
text-decoration: none; text-decoration: none;

5
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -1349,12 +1349,13 @@ function article_db_open($database) {
$dbh->exec("CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5( $dbh->exec("CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5(
newsgroup, newsgroup,
number, number,
date,
msgid, msgid,
date,
name,
subject, subject,
search_snippet)"); search_snippet)");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_insert AFTER INSERT ON articles BEGIN $dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_insert AFTER INSERT ON articles BEGIN
INSERT INTO search_fts(newsgroup, number, date, msgid, subject, search_snippet) VALUES(new.newsgroup, new.number, new.date, new.msgid, new.subject, new.search_snippet); INSERT INTO search_fts(newsgroup, number, msgid, date, name, subject, search_snippet) VALUES(new.newsgroup, new.number, new.msgid, new.date, new.name, new.subject, new.search_snippet);
END;"); END;");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_delete AFTER DELETE ON articles BEGIN $dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_delete AFTER DELETE ON articles BEGIN
DELETE FROM search_fts WHERE msgid = old.msgid; DELETE FROM search_fts WHERE msgid = old.msgid;

128
Rocksolid_Light/rocksolid/search.php
View File

@ -47,9 +47,12 @@ if ($_GET['searchpoint'] == 'Poster') {
echo '<td><input type="radio" name="searchpoint" value="subject" checked="checked"/>Subject&nbsp;'; echo '<td><input type="radio" name="searchpoint" value="subject" checked="checked"/>Subject&nbsp;';
echo '<input type="radio" name="searchpoint" value="name"/>Poster&nbsp;'; echo '<input type="radio" name="searchpoint" value="name"/>Poster&nbsp;';
} }
echo '<input type="radio" name="searchpoint" value="msgid"/>Message-ID';
if($CONFIG['article_database'] == '1') {
echo '&nbsp;<input type="radio" name="searchpoint" value="body"/>Body';
}
echo '</td></tr>';
?> ?>
<input type="radio" name="searchpoint" value="msgid"/>Message-ID</td>
</tr>
<tr> <tr>
<td><input name="command" type="hidden" id="command" value="Search" readonly="readonly"></td> <td><input name="command" type="hidden" id="command" value="Search" readonly="readonly"></td>
<?php echo '<input type="hidden" name="key" value="'.hash('md5', $admin['key']).'">';?> <?php echo '<input type="hidden" name="key" value="'.hash('md5', $admin['key']).'">';?>
@ -137,32 +140,16 @@ $results=0;
} else { } else {
$offset=$CONFIG['timezone']; $offset=$CONFIG['timezone'];
} }
$searchterms = "%".$_POST['terms']."%"; $overview = array();
# Prepare search database if($_POST['searchpoint'] == 'body') {
$database = $spooldir.'/articles-overview.db3'; $overview = get_body_search($group, $_POST['terms']);
$table = 'overview'; } else {
$dbh = rslight_db_open($database, $table); $overview = get_header_search($group, $_POST['terms']);
$overview = array(); }
if($dbh) { foreach($overview as $overviewline) {
if(is_multibyte($_POST['terms'])) {
$stmt = $dbh->query("SELECT * FROM $table");
while($row = $stmt->fetch()) {
if(stripos(quoted_printable_decode(mb_decode_mimeheader($row[$_POST['searchpoint']])), $_POST['terms']) !== false) {
$overview[] = $row;
}
}
} else {
$stmt = $dbh->prepare("SELECT * FROM $table WHERE ".$_POST['searchpoint']." like :terms ORDER BY date DESC");
$stmt->bindParam(':terms', $searchterms);
$stmt->execute();
while($found = $stmt->fetch()) {
$overview[] = $found;
}
}
$dbh = null;
foreach($overview as $overviewline) {
/* Find section for links */ /* Find section for links */
$menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); $menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($menulist as $menu) { foreach($menulist as $menu) {
if($menu[0] == '#') { if($menu[0] == '#') {
continue; continue;
@ -178,6 +165,8 @@ $results=0;
} }
} }
} }
fclose($glfp);
# Generate link # Generate link
$url = "../".$section."/article-flat.php?id=".$overviewline['number']."&group="._rawurlencode($overviewline['newsgroup'])."#".$overviewline['number']; $url = "../".$section."/article-flat.php?id=".$overviewline['number']."&group="._rawurlencode($overviewline['newsgroup'])."#".$overviewline['number'];
$groupurl = "../".$section."/thread.php?group="._rawurlencode($overviewline['newsgroup']); $groupurl = "../".$section."/thread.php?group="._rawurlencode($overviewline['newsgroup']);
@ -186,35 +175,61 @@ $results=0;
// Use local timezone if possible // Use local timezone if possible
$ts = new DateTime(date($text_header["date_format"], $overviewline['date']), new DateTimeZone('UTC')); $ts = new DateTime(date($text_header["date_format"], $overviewline['date']), new DateTimeZone('UTC'));
$ts->add(DateInterval::createFromDateString($offset.' minutes')); $ts->add(DateInterval::createFromDateString($offset.' minutes'));
if($offset != 0) { if($offset != 0) {
$newdate = $ts->format('D, j M Y H:i'); $newdate = $ts->format('D, j M Y H:i');
} else { } else {
$newdate = $ts->format($text_header["date_format"]); $newdate = $ts->format($text_header["date_format"]);
} }
unset($ts);
unset($ts);
$fromline=address_decode(headerDecode($overviewline['name']),"nirgendwo"); $fromline=address_decode(headerDecode($overviewline['name']),"nirgendwo");
if (!isset($fromline[0]["personal"])) { if (!isset($fromline[0]["personal"])) {
$lastname=$fromline[0]["mailbox"];; $lastname=$fromline[0]["mailbox"];;
} else { } else {
$lastname=$fromline[0]["personal"]; $lastname=$fromline[0]["personal"];
} }
if(($results % 2) != 0){ if(($results % 2) != 0){
echo '<tr class="np_result_line1"><td style="word-wrap:break-word";>'; echo '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
} else { } else {
echo '<tr class="np_result_line2"><td style="word-wrap:break-word";>'; echo '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
} }
echo '<p class=np_ob_subject>'; echo '<p class=np_ob_subject>';
echo '<b><a href="'.$url.'">'.mb_decode_mimeheader($overviewline['subject'])."</a></b>\r\n"; echo '<b><a href="'.$url.'">'.mb_decode_mimeheader($overviewline['subject'])."</a></b>\r\n";
echo '</p><p class=np_ob_group>'; echo '</p><p class=np_ob_group>';
echo '<a href="'.$groupurl.'">'.$overviewline['newsgroup'].'</a>'; echo '<a href="'.$groupurl.'">'.$overviewline['newsgroup'].'</a>';
echo '</p>'; echo '</p>';
echo '<p class=np_ob_posted_date>Posted: '.$newdate.' by: '.mb_decode_mimeheader($overviewline['name']).'</p>';
$articlefrom[0] = $overviewline['name'];
$fromoutput = explode("<", html_entity_decode($articlefrom[0]));
// Just an email address?
if(strlen($fromoutput[0]) < 2) {
preg_match("/\<([^\)]*)\@/", html_entity_decode($articlefrom[0]), $fromaddress);
$fromoutput[0] = $fromaddress[1];
}
if(strpos($fromoutput[0], "(")) {
preg_match("/\(([^\)]*)\)/", html_entity_decode($articlefrom[0]), $fromaddress);
$fromoutput[0] = $fromaddress[1];
}
if((isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) && (strpos($fromoutput[0], '@') !== false)) {
$poster_name = truncate_email($fromoutput[0]);
} else {
$poster_name = $fromoutput[0];
}
$poster_name = trim($poster_name, "\"");
echo '<p class=np_ob_posted_date>Posted: '.$newdate.' by: '.create_name_link(mb_decode_mimeheader(mb_decode_mimeheader($poster_name))).'</p>';
if($_POST['searchpoint'] == 'body') {
echo $overviewline['snippet'];
}
echo '</td></tr>'; echo '</td></tr>';
if($results++ > ($maxdisplay - 2)) if($results++ > ($maxdisplay - 2))
break; break;
} // }
} }
echo '</table>'; echo '</table>';
@ -228,6 +243,57 @@ ob_end_clean();
echo $thispage; echo $thispage;
function get_body_search($group, $terms) {
GLOBAL $CONFIG, $config_name, $spooldir;
//$group = 'rocksolid.nodes';
// $terms = "%".$terms."%";
$local_groupfile=$spooldir."/".$config_name."/local_groups.txt";
$grouplist = file($local_groupfile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($grouplist as $thisgroup) {
$name = explode(':', $thisgroup);
$group=$name[0];
$database = $spooldir.'/'.$group.'-articles.db3';
$dbh = article_db_open($database);
$stmt = $dbh->prepare("SELECT snippet(search_fts, 6, '<strong><font class=search_result><i>', '</i></font></strong>', '...', 50) as snippet, newsgroup, number, name, date, subject FROM search_fts WHERE search_snippet MATCH :terms ORDER BY rank");
$stmt->bindParam(':terms', $terms);
$stmt->execute();
while ($row = $stmt->fetch()) {
$overview[] = $row;
}
$dbh = null;
}
return $overview;
}
function get_header_search($group, $terms) {
GLOBAL $CONFIG, $spooldir;
$searchterms = "%".$terms."%";
# Prepare search database
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
if($dbh) {
if(is_multibyte($_POST['terms'])) {
$stmt = $dbh->query("SELECT * FROM $table");
while($row = $stmt->fetch()) {
if(stripos(quoted_printable_decode(mb_decode_mimeheader($row[$_POST['searchpoint']])), $_POST['terms']) !== false) {
$overview[] = $row;
}
}
} else {
$stmt = $dbh->prepare("SELECT * FROM $table WHERE ".$_POST['searchpoint']." like :terms ORDER BY date DESC");
$stmt->bindParam(':terms', $searchterms);
$stmt->execute();
while($found = $stmt->fetch()) {
$overview[] = $found;
}
}
$dbh = null;
}
return $overview;
}
function highlightStr($haystack, $needle) { function highlightStr($haystack, $needle) {
preg_match_all("/$needle+/i", $haystack, $matches); preg_match_all("/$needle+/i", $haystack, $matches);
if (is_array($matches[0]) && count($matches[0]) >= 1) { if (is_array($matches[0]) && count($matches[0]) >= 1) {

31
Rocksolid_Light/rslight/scripts/import-db3.php
View File

@ -35,7 +35,19 @@ if (posix_getsid($pid) === false || !is_file($lockfile)) {
} }
$group = trim($argv[1]); $group = trim($argv[1]);
import_articles($group); if($group == '') {
$group_files = scandir($workpath);
foreach($group_files as $this_file) {
if(strpos($this_file, '-articles.db3') === false) {
continue;
}
$group = preg_replace('/-articles.db3/', '', $this_file);
echo 'Importing: '.$group."\n";
import_articles($group);
}
} else {
import_articles($group);
}
echo "\nImport Done\r\n"; echo "\nImport Done\r\n";
function import_articles($group) { function import_articles($group) {
@ -43,6 +55,9 @@ function import_articles($group) {
$overview_file = $workpath.'/'.$group."-overview"; $overview_file = $workpath.'/'.$group."-overview";
# Prepare databases # Prepare databases
// Overview db // Overview db
$new_article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3-new');
$new_article_sql = 'INSERT INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$new_article_stmt = $new_article_dbh->prepare($new_article_sql);
$database = $spooldir.'/articles-overview.db3'; $database = $spooldir.'/articles-overview.db3';
$table = 'overview'; $table = 'overview';
$dbh = rslight_db_open($database, $table); $dbh = rslight_db_open($database, $table);
@ -64,6 +79,7 @@ function import_articles($group) {
$ref=0; $ref=0;
$banned=0; $banned=0;
$is_header=1; $is_header=1;
$body="";
foreach($this_article as $response) foreach($this_article as $response)
{ {
$bytes = $bytes + mb_strlen($response, '8bit'); $bytes = $bytes + mb_strlen($response, '8bit');
@ -91,6 +107,10 @@ function import_articles($group) {
$xref=$response; $xref=$response;
$ref=0; $ref=0;
} }
if(stripos($response, "Content-Type: ") === 0) {
preg_match('/.*charset=.*/', $response, $te);
$content_type = explode("Content-Type: text/plain; charset=", $te[0]);
}
if(stripos($response, "References: ") === 0) { if(stripos($response, "References: ") === 0) {
$this_references=explode('References: ', $response); $this_references=explode('References: ', $response);
$references = $this_references[1]; $references = $this_references[1];
@ -102,11 +122,17 @@ function import_articles($group) {
} }
} }
$response=str_replace("\n","",str_replace("\r","",$response)); $response=str_replace("\n","",str_replace("\r","",$response));
} else {
$body.=$response."\n";
} }
} }
$lines=$lines-1; $lines=$lines-1;
$bytes = $bytes + ($lines * 2); $bytes = $bytes + ($lines * 2);
// add to database // add to database
// CREATE SEARCH SNIPPET
$this_snippet = get_search_snippet($body, $content_type[1]);
$new_article_stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1], $row['article'], $this_snippet]);
$stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]); $stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]);
file_put_contents($overview_file, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n", FILE_APPEND); file_put_contents($overview_file, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n", FILE_APPEND);
echo "\nImported: ".$group." ".$local; echo "\nImported: ".$group." ".$local;
@ -114,7 +140,10 @@ function import_articles($group) {
$i++; $i++;
$references=""; $references="";
} }
$new_article_dbh = null;
$article_dbh = null; $article_dbh = null;
$dbh = null; $dbh = null;
rename($spooldir.'/'.$group.'-articles.db3', $spooldir.'/'.$group.'-articles.db3-old');
rename($spooldir.'/'.$group.'-articles.db3-new', $spooldir.'/'.$group.'-articles.db3');
} }
?> ?>