novabbs
/
rocksolid-light
2
0
Fork 1
Code Issues 15 Pull Requests Releases Wiki Activity

Remove flat file overview and use articles-overview.db3 for all overview features.

This commit is contained in:
Retro_Guy 2023-08-13 05:08:04 -07:00
parent da6a4f5042
commit ed31b5b6a1
5 changed files with 118 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -1462,8 +1462,13 @@ function rslight_db_open($database, $table='overview') {
number TEXT, number TEXT,
msgid TEXT, msgid TEXT,
date TEXT, date TEXT,
datestring TEXT,
name TEXT, name TEXT,
subject TEXT, subject TEXT,
refs TEXT,
bytes TEXT,
lines TEXT,
xref TEXT,
unique (newsgroup, msgid))"); unique (newsgroup, msgid))");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)'); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)');
$stmt->execute(); $stmt->execute();

36
Rocksolid_Light/rslight/scripts/expire.php
View File

@ -61,35 +61,17 @@ file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group."
$articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]); $articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$articles_dbh = null; $articles_dbh = null;
} }
} } else { // Expire tradspool
$database = $spooldir.'/articles-overview.db3';
echo "Expiring group overview file...\n"; $dbh = rslight_db_open($database);
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring group overview file...", FILE_APPEND); $query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
$grouppath = preg_replace('/\./', '/', $group); $query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$this_overview=$spooldir.'/'.$group.'-overview'; $grouppath = preg_replace('/\./', '/', $group);
$out_overview=$this_overview.'.new'; while($row = $query->fetch()) {
$overviewfp=fopen($this_overview, 'r'); unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
$out_overviewfp=fopen($out_overview, 'w');
while($line=fgets($overviewfp)) {
$break=explode("\t", $line);
if(strtotime($break[3]) < $expireme) {
echo "Expiring: ".$break[4]." IN: ".$group." #".$break[0]."\r\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring: ".$break[4], FILE_APPEND);
// Remove article from tradspool:
if(is_file($spooldir.'/articles/'.$grouppath.'/'.$break[0])) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]);
} }
thread_cache_removearticle($group,$break[4]); $dbh = null;
continue;
} else {
fputs($out_overviewfp, $line);
}
} }
fclose($overviewfp);
fclose($out_overviewfp);
rename($out_overview, $this_overview);
chown($this_overview, $CONFIG['webserver_user']);
chgrp($this_overview, $webserver_group);
} }
unlink($lockfile); unlink($lockfile);
touch($spooldir.'/'.$config_name.'-expire-timer'); touch($spooldir.'/'.$config_name.'-expire-timer');

37
Rocksolid_Light/rslight/scripts/nocem.php
View File

@ -107,30 +107,19 @@ function delete_message($messageid, $group) {
$articles_dbh = null; $articles_dbh = null;
} }
} }
$this_overview=$spooldir.'/'.$group.'-overview';
if(false === (is_file($this_overview))) { // Tradspool
return; if($CONFIG['article_database'] != '1') {
} $database = $spooldir.'/articles-overview.db3';
$out_overview=$this_overview.'.new'; $dbh = rslight_db_open($database);
$overviewfp=fopen($this_overview, 'r'); $query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND msgid<:msgid');
$out_overviewfp=fopen($out_overview, 'w'); $query->execute([':newsgroup' => $group, ':msgid' => $messageid]);
while($line=fgets($overviewfp)) { $grouppath = preg_replace('/\./', '/', $group);
$break=explode("\t", $line); while($row = $query->fetch()) {
if($break[4] == $messageid) { unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
echo "DELETING: ".$messageid." IN: ".$group." #".$break[0]."\r\n"; }
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." DELETING: ".$messageid." IN: ".$group." #".$break[0], FILE_APPEND); $dbh = null;
$grouppath = preg_replace('/\./', '/', $group); }
unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]);
continue;
} else {
fputs($out_overviewfp, $line);
}
}
fclose($overviewfp);
fclose($out_overviewfp);
rename($out_overview, $this_overview);
chown($this_overview, $CONFIG['webserver_user']);
chgrp($this_overview, $webserver_group);
delete_message_from_overboard($config_name, $group, $messageid); delete_message_from_overboard($config_name, $group, $messageid);
return; return;
} }

113
Rocksolid_Light/rslight/scripts/rslight-lib.php
View File

@ -252,11 +252,14 @@ set_time_limit(0);
} }
function prepare_post($filename) { function prepare_post($filename) {
global $logdir; global $logdir, $spooldir;
$logfile = $logdir.'/nntp.log'; $logfile = $logdir.'/nntp.log';
$message = file($filename, FILE_IGNORE_NEW_LINES); $message = file($filename, FILE_IGNORE_NEW_LINES);
$lines = 0; $lines = 0;
$is_header = 1; $is_header = 1;
$nocem_check="@@NCM";
$bbsmail_check="@@RSL";
foreach($message as $line) { foreach($message as $line) {
if(trim($line) == "" || $lines > 0) { if(trim($line) == "" || $lines > 0) {
$is_header=0; $is_header=0;
@ -269,7 +272,13 @@ function prepare_post($filename) {
$ngroups=explode(': ', $line); $ngroups=explode(': ', $line);
$newsgroups=$ngroups[1]; $newsgroups=$ngroups[1];
$lines++; $lines++;
break; continue;
}
if(stripos($line, "Subject: ") === 0) {
$sub=explode(': ', $line);
$subject=$sub[1];
$lines++;
continue;
} }
} }
$ngroups = preg_split("/(\ |\,)/", trim($newsgroups)); $ngroups = preg_split("/(\ |\,)/", trim($newsgroups));
@ -283,6 +292,17 @@ function prepare_post($filename) {
} }
} }
if($ok == 1) { if($ok == 1) {
if((strpos($rslight_gpg['nntp_group'], $group) !== false) && ($rslight_gpg['enable'] == '1')) {
if(strpos($subject, $bbsmail_check) !== false) {
$bbsmail_file = preg_replace('/@@RSL /', '', $subject);
$bbsmail_filename = $spooldir."/bbsmail/in/bbsmail-".$bbsmail_file;
copy($filename, $bbsmail_filename);
}
}
if(strpos($subject, $nocem_check) !== false) {
$nocem_file = tempnam($spooldir."/nocem", "nocem-".$group."-");
copy($filename, $nocem_file);
}
$response="240 Article received OK\r\n"; $response="240 Article received OK\r\n";
} else { } else {
$response="441 Posting failed\r\n"; $response="441 Posting failed\r\n";
@ -433,25 +453,25 @@ function process_post($message, $group) {
if($response == "") { if($response == "") {
// Check for duplicate msgid // Check for duplicate msgid
$duplicate=0; $duplicate=0;
if(file_exists($spooldir."/".$group."-overview")) { $database = $spooldir.'/articles-overview.db3';
$group_overviewfp=fopen($spooldir."/".$group."-overview", 'r'); $table = 'overview';
while($group_overview=fgets($group_overviewfp, 2048)) { $dbh = rslight_db_open($database, $table);
$overview_msgid = explode("\t", $group_overview); $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND msgid=:msgid ORDER BY number");
if(strpos($overview_msgid[4], $msgid) !== false) { $stmt->execute(['thisgroup' => $group, ':msgid' => $msgid]);
unlink($postfilename); while($found = $stmt->fetch()) {
unlink($postfilename);
file_put_contents($logfile, "\n".format_log_date()." ".$section." Duplicate Message-ID for: ".$msgid, FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$section." Duplicate Message-ID for: ".$msgid, FILE_APPEND);
$duplicate=1; $duplicate=1;
break; break;
} }
} $dbh = null;
fclose($group_overviewfp);
} if($duplicate == 0) {
if($duplicate == 0) { $response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body);
$response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body); } else {
} else { $response="441 Posting failed (duplicate)\r\n";
$response="441 Posting failed (duplicate)\r\n"; }
} }
}
} }
return $response; return $response;
} }
@ -610,7 +630,7 @@ function get_title($mode) {
} }
function get_xover($articles, $msgsock) { function get_xover($articles, $msgsock) {
global $nntp_group,$nntp_article,$workpath,$path; global $nntp_group,$nntp_article,$workpath,$path,$spooldir;
// Use article pointer // Use article pointer
if(!isset($articles) && is_numeric($nntp_article)) { if(!isset($articles) && is_numeric($nntp_article)) {
$articles = $nntp_article; $articles = $nntp_article;
@ -637,7 +657,6 @@ function get_xover($articles, $msgsock) {
$msg="420 no article(s) selected\r\n"; $msg="420 no article(s) selected\r\n";
return $msg; return $msg;
} }
$overviewfile=$workpath.$nntp_group."-overview";
if(!isset($this_id)) { if(!isset($this_id)) {
$article_num = explode('-', $articles); $article_num = explode('-', $articles);
$first = $article_num[0]; $first = $article_num[0];
@ -660,19 +679,17 @@ function get_xover($articles, $msgsock) {
} }
} }
fwrite($msgsock, $output, strlen($output)); fwrite($msgsock, $output, strlen($output));
if(file_exists($overviewfile)) {
$overviewfp=fopen($overviewfile, 'r'); $database = $spooldir.'/articles-overview.db3';
while($overviewline=fgets($overviewfp)) { $table = 'overview';
$article=preg_split("/[\s,]+/", $overviewline); $dbh = rslight_db_open($database, $table);
for($i=$first; $i<=$last; $i++) { $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND number BETWEEN :first AND :last ORDER BY number");
if($article[0] === strval($i)) { $stmt->execute(['thisgroup' => $nntp_group, ':first' => $first, ':last' => $last]);
$overviewline = trim($overviewline)."\r\n"; $msg = '';
fwrite($msgsock, $overviewline, strlen($overviewline)); while($row = $stmt->fetch()) {
} $msg.= $row['number']."\t".$row['subject']."\t".$row['name']."\t".$row['datestring']."\t".$row['msgid']."\t".$row['refs']."\t".$row['bytes']."\t".$row['lines']."\t".$row['xref']."\r\n";
}
}
fclose($overviewfp);
} }
$dbh = null;
$msg.=".\r\n"; $msg.=".\r\n";
return $msg; return $msg;
} }
@ -691,18 +708,25 @@ function get_stat($article) {
$msg="423 No article number selected\r\n"; $msg="423 No article number selected\r\n";
return $msg; return $msg;
} }
$overviewfile=$workpath.$nntp_group."-overview";
$overviewfp=fopen($overviewfile, 'r'); $database = $spooldir.'/articles-overview.db3';
while($overviewline=fgets($overviewfp)) { if(!is_file($database)) {
$over=explode("\t", $overviewline); return false;
if(trim($over[0]) == trim($article)) { }
$msg="223 ".$article." ".$over[4]." status\r\n"; $dbh = rslight_db_open($database);
fclose($overviewfp); $query = $articles_dbh->prepare('SELECT * FROM overview WHERE number=:number AND newsgroup=:newsgroup');
return $msg; $query->execute(['number' => $article, 'newsgroup' => $nntp_group]);
} $found = 0;
while ($row = $query->fetch()) {
$found = 1;
break;
}
$dbh = null;
if($found == 1) {
$msg="223 ".$article." ".$row['msgid']." status\r\n";
} else {
$msg="423 No such article number ".$article."\r\n";
} }
fclose($overviewfp);
$msg="423 No such article number ".$article."\r\n";
return $msg; return $msg;
} }
@ -1112,7 +1136,6 @@ $date_i,$mid_i,$references_i,$bytes_i,$lines_i,$xref_i,$body) {
touch($tmp_file, $article_date); touch($tmp_file, $article_date);
file_put_contents($logfile, "\n".format_log_date()." ".$section." Inserting local post: ".$nntp_group.":".$local, FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$section." Inserting local post: ".$nntp_group.":".$local, FILE_APPEND);
// Overview // Overview
$overviewHandle = fopen($spooldir."/".$nntp_group."-overview", 'a');
# Prepare overview database # Prepare overview database
$database = $spooldir.'/articles-overview.db3'; $database = $spooldir.'/articles-overview.db3';
$table = 'overview'; $table = 'overview';

46
Rocksolid_Light/rslight/scripts/spoolnews.php
View File

@ -38,7 +38,7 @@ if(!isset($maxarticles_per_run)) {
$maxarticles_per_run = 100; $maxarticles_per_run = 100;
} }
if(!isset($maxfirstrequest)) { if(!isset($maxfirstrequest)) {
$maxfirstrequest = 1000; $maxfirstrequest = 100;
} }
if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) { if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) {
@ -198,30 +198,34 @@ function get_articles($ns, $group) {
if($article < $detail[2]) { if($article < $detail[2]) {
$article = $detail[2]; $article = $detail[2];
} }
// Broken message on last run? Let's try again.
/* // Articles Database
if($article > ($detail[3])) {
$article = $detail[3];
}
*/
# Prepare databases
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$sql = 'INSERT OR IGNORE INTO '.$table.'(newsgroup, number, msgid, date, name, subject) VALUES(?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql);
if($CONFIG['article_database'] == '1') { if($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3'); $article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3');
$article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)'; $article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql); $article_stmt = $article_dbh->prepare($article_sql);
} }
// Create list of message-ids // Create list of message-ids
$group_overviewfile = $spooldir."/".$group."-overview"; $database = $spooldir.'/articles-overview.db3';
$gover = file($group_overviewfile); $table = 'overview';
foreach($gover as $group_overview) { $dbh = rslight_db_open($database, $table);
$overview_msgid = explode("\t", $group_overview); $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup");
$msgids[trim($overview_msgid[4])] = true; $stmt->bindParam(':newsgroup', $nntp_group);
$stmt->execute();
while($row = $stmt->fetch()) {
$msgids[$row['msgid']] = true;
break;
} }
$dbh = null;
// Overview database
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$sql = 'INSERT OR IGNORE INTO overview(newsgroup, number, msgid, date, datestring, name, subject, refs, bytes, lines, xref) VALUES(?,?,?,?,?,?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql);
// Get overview from server // Get overview from server
$server_overview = array(); $server_overview = array();
$re = false; $re = false;
@ -370,12 +374,8 @@ function get_articles($ns, $group) {
} }
} }
// Overview // Overview
$overviewHandle = fopen($workpath.$group."-overview", 'a'); $stmt->execute([$group, $local, $mid[1], $article_date, $finddate[1], $from[1], $subject[1], $references, $bytes, $lines, $xref]);
fputs($overviewHandle, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n");
fclose($overviewHandle);
$references=""; $references="";
// add to database
$stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]);
if($CONFIG['article_database'] == '1') { if($CONFIG['article_database'] == '1') {
$this_article = file_get_contents($grouppath."/".$local); $this_article = file_get_contents($grouppath."/".$local);
// CREATE SEARCH SNIPPET // CREATE SEARCH SNIPPET