Add body search feature per section

This commit is contained in:
Retro_Guy 2021-03-10 23:55:01 -07:00
parent c6400cd791
commit c06a916037
4 changed files with 134 additions and 34 deletions

View File

@ -71,6 +71,10 @@
color: var(--color-visited);
}
.search_result {
color: var(--color-text-alt);
}
/* Title */
.np_title {
text-decoration: none;

View File

@ -1349,12 +1349,13 @@ function article_db_open($database) {
$dbh->exec("CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5(
newsgroup,
number,
date,
msgid,
date,
name,
subject,
search_snippet)");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_insert AFTER INSERT ON articles BEGIN
INSERT INTO search_fts(newsgroup, number, date, msgid, subject, search_snippet) VALUES(new.newsgroup, new.number, new.date, new.msgid, new.subject, new.search_snippet);
INSERT INTO search_fts(newsgroup, number, msgid, date, name, subject, search_snippet) VALUES(new.newsgroup, new.number, new.msgid, new.date, new.name, new.subject, new.search_snippet);
END;");
$dbh->exec("CREATE TRIGGER IF NOT EXISTS after_articles_delete AFTER DELETE ON articles BEGIN
DELETE FROM search_fts WHERE msgid = old.msgid;

View File

@ -47,9 +47,12 @@ if ($_GET['searchpoint'] == 'Poster') {
echo '<td><input type="radio" name="searchpoint" value="subject" checked="checked"/>Subject&nbsp;';
echo '<input type="radio" name="searchpoint" value="name"/>Poster&nbsp;';
}
echo '<input type="radio" name="searchpoint" value="msgid"/>Message-ID';
if($CONFIG['article_database'] == '1') {
echo '&nbsp;<input type="radio" name="searchpoint" value="body"/>Body';
}
echo '</td></tr>';
?>
<input type="radio" name="searchpoint" value="msgid"/>Message-ID</td>
</tr>
<tr>
<td><input name="command" type="hidden" id="command" value="Search" readonly="readonly"></td>
<?php echo '<input type="hidden" name="key" value="'.hash('md5', $admin['key']).'">';?>
@ -137,12 +140,139 @@ $results=0;
} else {
$offset=$CONFIG['timezone'];
}
$searchterms = "%".$_POST['terms']."%";
$overview = array();
if($_POST['searchpoint'] == 'body') {
$overview = get_body_search($group, $_POST['terms']);
} else {
$overview = get_header_search($group, $_POST['terms']);
}
foreach($overview as $overviewline) {
/* Find section for links */
$menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($menulist as $menu) {
if($menu[0] == '#') {
continue;
}
$menuitem=explode(':', $menu);
$glfp=fopen($config_dir.$menuitem[0]."/groups.txt", 'r');
$section="";
while($gl=fgets($glfp)) {
$group_name = preg_split("/( |\t)/", $gl, 2);
if(stripos(trim($overviewline['newsgroup']), trim($group_name[0])) !== false) {
$section=$menuitem[0];
break 2;
}
}
}
fclose($glfp);
# Generate link
$url = "../".$section."/article-flat.php?id=".$overviewline['number']."&group="._rawurlencode($overviewline['newsgroup'])."#".$overviewline['number'];
$groupurl = "../".$section."/thread.php?group="._rawurlencode($overviewline['newsgroup']);
$fromoutput = explode("<", html_entity_decode($overviewline['name']));
// Use local timezone if possible
$ts = new DateTime(date($text_header["date_format"], $overviewline['date']), new DateTimeZone('UTC'));
$ts->add(DateInterval::createFromDateString($offset.' minutes'));
if($offset != 0) {
$newdate = $ts->format('D, j M Y H:i');
} else {
$newdate = $ts->format($text_header["date_format"]);
}
unset($ts);
$fromline=address_decode(headerDecode($overviewline['name']),"nirgendwo");
if (!isset($fromline[0]["personal"])) {
$lastname=$fromline[0]["mailbox"];;
} else {
$lastname=$fromline[0]["personal"];
}
if(($results % 2) != 0){
echo '<tr class="np_result_line1"><td class="np_result_line1" style="word-wrap:break-word";>';
} else {
echo '<tr class="np_result_line2"><td class="np_result_line2" style="word-wrap:break-word";>';
}
echo '<p class=np_ob_subject>';
echo '<b><a href="'.$url.'">'.mb_decode_mimeheader($overviewline['subject'])."</a></b>\r\n";
echo '</p><p class=np_ob_group>';
echo '<a href="'.$groupurl.'">'.$overviewline['newsgroup'].'</a>';
echo '</p>';
$articlefrom[0] = $overviewline['name'];
$fromoutput = explode("<", html_entity_decode($articlefrom[0]));
// Just an email address?
if(strlen($fromoutput[0]) < 2) {
preg_match("/\<([^\)]*)\@/", html_entity_decode($articlefrom[0]), $fromaddress);
$fromoutput[0] = $fromaddress[1];
}
if(strpos($fromoutput[0], "(")) {
preg_match("/\(([^\)]*)\)/", html_entity_decode($articlefrom[0]), $fromaddress);
$fromoutput[0] = $fromaddress[1];
}
if((isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) && (strpos($fromoutput[0], '@') !== false)) {
$poster_name = truncate_email($fromoutput[0]);
} else {
$poster_name = $fromoutput[0];
}
$poster_name = trim($poster_name, "\"");
echo '<p class=np_ob_posted_date>Posted: '.$newdate.' by: '.create_name_link(mb_decode_mimeheader(mb_decode_mimeheader($poster_name))).'</p>';
if($_POST['searchpoint'] == 'body') {
echo $overviewline['snippet'];
}
echo '</td></tr>';
if($results++ > ($maxdisplay - 2))
break;
// }
}
echo '</table>';
echo "<p class=np_ob_tail><b>".$results."</b> matching articles found.</p>\r\n";
#echo "<center><i>Rocksolid Overboard</i> version ".$version;
include "tail.inc";
$thispage = ob_get_contents();
ob_end_clean();
echo $thispage;
function get_body_search($group, $terms) {
GLOBAL $CONFIG, $config_name, $spooldir;
//$group = 'rocksolid.nodes';
// $terms = "%".$terms."%";
$local_groupfile=$spooldir."/".$config_name."/local_groups.txt";
$grouplist = file($local_groupfile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($grouplist as $thisgroup) {
$name = explode(':', $thisgroup);
$group=$name[0];
$database = $spooldir.'/'.$group.'-articles.db3';
$dbh = article_db_open($database);
$stmt = $dbh->prepare("SELECT snippet(search_fts, 6, '<strong><font class=search_result><i>', '</i></font></strong>', '...', 50) as snippet, newsgroup, number, name, date, subject FROM search_fts WHERE search_snippet MATCH :terms ORDER BY rank");
$stmt->bindParam(':terms', $terms);
$stmt->execute();
while ($row = $stmt->fetch()) {
$overview[] = $row;
}
$dbh = null;
}
return $overview;
}
function get_header_search($group, $terms) {
GLOBAL $CONFIG, $spooldir;
$searchterms = "%".$terms."%";
# Prepare search database
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$overview = array();
if($dbh) {
if(is_multibyte($_POST['terms'])) {
$stmt = $dbh->query("SELECT * FROM $table");
@ -160,74 +290,10 @@ $results=0;
}
}
$dbh = null;
foreach($overview as $overviewline) {
/* Find section for links */
$menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($menulist as $menu) {
if($menu[0] == '#') {
continue;
}
$menuitem=explode(':', $menu);
$glfp=fopen($config_dir.$menuitem[0]."/groups.txt", 'r');
$section="";
while($gl=fgets($glfp)) {
$group_name = preg_split("/( |\t)/", $gl, 2);
if(stripos(trim($overviewline['newsgroup']), trim($group_name[0])) !== false) {
$section=$menuitem[0];
break 2;
}
}
}
# Generate link
$url = "../".$section."/article-flat.php?id=".$overviewline['number']."&group="._rawurlencode($overviewline['newsgroup'])."#".$overviewline['number'];
$groupurl = "../".$section."/thread.php?group="._rawurlencode($overviewline['newsgroup']);
$fromoutput = explode("<", html_entity_decode($overviewline['name']));
// Use local timezone if possible
$ts = new DateTime(date($text_header["date_format"], $overviewline['date']), new DateTimeZone('UTC'));
$ts->add(DateInterval::createFromDateString($offset.' minutes'));
if($offset != 0) {
$newdate = $ts->format('D, j M Y H:i');
} else {
$newdate = $ts->format($text_header["date_format"]);
}
unset($ts);
$fromline=address_decode(headerDecode($overviewline['name']),"nirgendwo");
if (!isset($fromline[0]["personal"])) {
$lastname=$fromline[0]["mailbox"];;
} else {
$lastname=$fromline[0]["personal"];
}
if(($results % 2) != 0){
echo '<tr class="np_result_line1"><td style="word-wrap:break-word";>';
} else {
echo '<tr class="np_result_line2"><td style="word-wrap:break-word";>';
}
echo '<p class=np_ob_subject>';
echo '<b><a href="'.$url.'">'.mb_decode_mimeheader($overviewline['subject'])."</a></b>\r\n";
echo '</p><p class=np_ob_group>';
echo '<a href="'.$groupurl.'">'.$overviewline['newsgroup'].'</a>';
echo '</p>';
echo '<p class=np_ob_posted_date>Posted: '.$newdate.' by: '.mb_decode_mimeheader($overviewline['name']).'</p>';
echo '</td></tr>';
if($results++ > ($maxdisplay - 2))
break;
}
return $overview;
}
echo '</table>';
echo "<p class=np_ob_tail><b>".$results."</b> matching articles found.</p>\r\n";
#echo "<center><i>Rocksolid Overboard</i> version ".$version;
include "tail.inc";
$thispage = ob_get_contents();
ob_end_clean();
echo $thispage;
function highlightStr($haystack, $needle) {
preg_match_all("/$needle+/i", $haystack, $matches);
if (is_array($matches[0]) && count($matches[0]) >= 1) {

View File

@ -35,7 +35,19 @@ if (posix_getsid($pid) === false || !is_file($lockfile)) {
}
$group = trim($argv[1]);
import_articles($group);
if($group == '') {
$group_files = scandir($workpath);
foreach($group_files as $this_file) {
if(strpos($this_file, '-articles.db3') === false) {
continue;
}
$group = preg_replace('/-articles.db3/', '', $this_file);
echo 'Importing: '.$group."\n";
import_articles($group);
}
} else {
import_articles($group);
}
echo "\nImport Done\r\n";
function import_articles($group) {
@ -43,6 +55,9 @@ function import_articles($group) {
$overview_file = $workpath.'/'.$group."-overview";
# Prepare databases
// Overview db
$new_article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3-new');
$new_article_sql = 'INSERT INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$new_article_stmt = $new_article_dbh->prepare($new_article_sql);
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
@ -64,6 +79,7 @@ function import_articles($group) {
$ref=0;
$banned=0;
$is_header=1;
$body="";
foreach($this_article as $response)
{
$bytes = $bytes + mb_strlen($response, '8bit');
@ -91,6 +107,10 @@ function import_articles($group) {
$xref=$response;
$ref=0;
}
if(stripos($response, "Content-Type: ") === 0) {
preg_match('/.*charset=.*/', $response, $te);
$content_type = explode("Content-Type: text/plain; charset=", $te[0]);
}
if(stripos($response, "References: ") === 0) {
$this_references=explode('References: ', $response);
$references = $this_references[1];
@ -102,11 +122,17 @@ function import_articles($group) {
}
}
$response=str_replace("\n","",str_replace("\r","",$response));
} else {
$body.=$response."\n";
}
}
$lines=$lines-1;
$bytes = $bytes + ($lines * 2);
// add to database
// CREATE SEARCH SNIPPET
$this_snippet = get_search_snippet($body, $content_type[1]);
$new_article_stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1], $row['article'], $this_snippet]);
$stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]);
file_put_contents($overview_file, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n", FILE_APPEND);
echo "\nImported: ".$group." ".$local;
@ -114,7 +140,10 @@ function import_articles($group) {
$i++;
$references="";
}
$new_article_dbh = null;
$article_dbh = null;
$dbh = null;
rename($spooldir.'/'.$group.'-articles.db3', $spooldir.'/'.$group.'-articles.db3-old');
rename($spooldir.'/'.$group.'-articles.db3-new', $spooldir.'/'.$group.'-articles.db3');
}
?>