novabbs
/
rocksolid-light
3
0
Fork 1
Code Issues 22 Pull Requests Releases Wiki Activity

Remove flat file overview and use articles-overview.db3 for all overview features.

This commit is contained in:
Retro_Guy 2023-08-13 05:08:04 -07:00
parent da6a4f5042
commit ed31b5b6a1
5 changed files with 118 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -1462,8 +1462,13 @@ function rslight_db_open($database, $table='overview') {
number TEXT,
msgid TEXT,
date TEXT,
datestring TEXT,
name TEXT,
subject TEXT,
refs TEXT,
bytes TEXT,
lines TEXT,
xref TEXT,
unique (newsgroup, msgid))");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)');
$stmt->execute();

36
Rocksolid_Light/rslight/scripts/expire.php
View File

@ -61,35 +61,17 @@ file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group."
$articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$articles_dbh = null;
}
}
echo "Expiring group overview file...\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring group overview file...", FILE_APPEND);
$grouppath = preg_replace('/\./', '/', $group);
$this_overview=$spooldir.'/'.$group.'-overview';
$out_overview=$this_overview.'.new';
$overviewfp=fopen($this_overview, 'r');
$out_overviewfp=fopen($out_overview, 'w');
while($line=fgets($overviewfp)) {
$break=explode("\t", $line);
if(strtotime($break[3]) < $expireme) {
echo "Expiring: ".$break[4]." IN: ".$group." #".$break[0]."\r\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring: ".$break[4], FILE_APPEND);
// Remove article from tradspool:
if(is_file($spooldir.'/articles/'.$grouppath.'/'.$break[0])) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]);
} else { // Expire tradspool
$database = $spooldir.'/articles-overview.db3';
$dbh = rslight_db_open($database);
$query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
$query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$grouppath = preg_replace('/\./', '/', $group);
while($row = $query->fetch()) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
}
thread_cache_removearticle($group,$break[4]);
continue;
} else {
fputs($out_overviewfp, $line);
}
$dbh = null;
}
fclose($overviewfp);
fclose($out_overviewfp);
rename($out_overview, $this_overview);
chown($this_overview, $CONFIG['webserver_user']);
chgrp($this_overview, $webserver_group);
}
unlink($lockfile);
touch($spooldir.'/'.$config_name.'-expire-timer');

37
Rocksolid_Light/rslight/scripts/nocem.php
View File

@ -107,30 +107,19 @@ function delete_message($messageid, $group) {
$articles_dbh = null;
}
}
$this_overview=$spooldir.'/'.$group.'-overview';
if(false === (is_file($this_overview))) {
return;
}
$out_overview=$this_overview.'.new';
$overviewfp=fopen($this_overview, 'r');
$out_overviewfp=fopen($out_overview, 'w');
while($line=fgets($overviewfp)) {
$break=explode("\t", $line);
if($break[4] == $messageid) {
echo "DELETING: ".$messageid." IN: ".$group." #".$break[0]."\r\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." DELETING: ".$messageid." IN: ".$group." #".$break[0], FILE_APPEND);
$grouppath = preg_replace('/\./', '/', $group);
unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]);
continue;
} else {
fputs($out_overviewfp, $line);
}
}
fclose($overviewfp);
fclose($out_overviewfp);
rename($out_overview, $this_overview);
chown($this_overview, $CONFIG['webserver_user']);
chgrp($this_overview, $webserver_group);
// Tradspool
if($CONFIG['article_database'] != '1') {
$database = $spooldir.'/articles-overview.db3';
$dbh = rslight_db_open($database);
$query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND msgid<:msgid');
$query->execute([':newsgroup' => $group, ':msgid' => $messageid]);
$grouppath = preg_replace('/\./', '/', $group);
while($row = $query->fetch()) {
unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']);
}
$dbh = null;
}
delete_message_from_overboard($config_name, $group, $messageid);
return;
}

113
Rocksolid_Light/rslight/scripts/rslight-lib.php
View File

@ -252,11 +252,14 @@ set_time_limit(0);
}
function prepare_post($filename) {
global $logdir;
global $logdir, $spooldir;
$logfile = $logdir.'/nntp.log';
$message = file($filename, FILE_IGNORE_NEW_LINES);
$lines = 0;
$is_header = 1;
$nocem_check="@@NCM";
$bbsmail_check="@@RSL";
foreach($message as $line) {
if(trim($line) == "" || $lines > 0) {
$is_header=0;
@ -269,7 +272,13 @@ function prepare_post($filename) {
$ngroups=explode(': ', $line);
$newsgroups=$ngroups[1];
$lines++;
break;
continue;
}
if(stripos($line, "Subject: ") === 0) {
$sub=explode(': ', $line);
$subject=$sub[1];
$lines++;
continue;
}
}
$ngroups = preg_split("/(\ |\,)/", trim($newsgroups));
@ -283,6 +292,17 @@ function prepare_post($filename) {
}
}
if($ok == 1) {
if((strpos($rslight_gpg['nntp_group'], $group) !== false) && ($rslight_gpg['enable'] == '1')) {
if(strpos($subject, $bbsmail_check) !== false) {
$bbsmail_file = preg_replace('/@@RSL /', '', $subject);
$bbsmail_filename = $spooldir."/bbsmail/in/bbsmail-".$bbsmail_file;
copy($filename, $bbsmail_filename);
}
}
if(strpos($subject, $nocem_check) !== false) {
$nocem_file = tempnam($spooldir."/nocem", "nocem-".$group."-");
copy($filename, $nocem_file);
}
$response="240 Article received OK\r\n";
} else {
$response="441 Posting failed\r\n";
@ -433,25 +453,25 @@ function process_post($message, $group) {
if($response == "") {
// Check for duplicate msgid
$duplicate=0;
if(file_exists($spooldir."/".$group."-overview")) {
$group_overviewfp=fopen($spooldir."/".$group."-overview", 'r');
while($group_overview=fgets($group_overviewfp, 2048)) {
$overview_msgid = explode("\t", $group_overview);
if(strpos($overview_msgid[4], $msgid) !== false) {
unlink($postfilename);
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND msgid=:msgid ORDER BY number");
$stmt->execute(['thisgroup' => $group, ':msgid' => $msgid]);
while($found = $stmt->fetch()) {
unlink($postfilename);
file_put_contents($logfile, "\n".format_log_date()." ".$section." Duplicate Message-ID for: ".$msgid, FILE_APPEND);
$duplicate=1;
break;
}
}
fclose($group_overviewfp);
}
if($duplicate == 0) {
$response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body);
} else {
$response="441 Posting failed (duplicate)\r\n";
$duplicate=1;
break;
}
$dbh = null;
if($duplicate == 0) {
$response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body);
} else {
$response="441 Posting failed (duplicate)\r\n";
}
}
}
}
return $response;
}
@ -610,7 +630,7 @@ function get_title($mode) {
}
function get_xover($articles, $msgsock) {
global $nntp_group,$nntp_article,$workpath,$path;
global $nntp_group,$nntp_article,$workpath,$path,$spooldir;
// Use article pointer
if(!isset($articles) && is_numeric($nntp_article)) {
$articles = $nntp_article;
@ -637,7 +657,6 @@ function get_xover($articles, $msgsock) {
$msg="420 no article(s) selected\r\n";
return $msg;
}
$overviewfile=$workpath.$nntp_group."-overview";
if(!isset($this_id)) {
$article_num = explode('-', $articles);
$first = $article_num[0];
@ -660,19 +679,17 @@ function get_xover($articles, $msgsock) {
}
}
fwrite($msgsock, $output, strlen($output));
if(file_exists($overviewfile)) {
$overviewfp=fopen($overviewfile, 'r');
while($overviewline=fgets($overviewfp)) {
$article=preg_split("/[\s,]+/", $overviewline);
for($i=$first; $i<=$last; $i++) {
if($article[0] === strval($i)) {
$overviewline = trim($overviewline)."\r\n";
fwrite($msgsock, $overviewline, strlen($overviewline));
}
}
}
fclose($overviewfp);
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND number BETWEEN :first AND :last ORDER BY number");
$stmt->execute(['thisgroup' => $nntp_group, ':first' => $first, ':last' => $last]);
$msg = '';
while($row = $stmt->fetch()) {
$msg.= $row['number']."\t".$row['subject']."\t".$row['name']."\t".$row['datestring']."\t".$row['msgid']."\t".$row['refs']."\t".$row['bytes']."\t".$row['lines']."\t".$row['xref']."\r\n";
}
$dbh = null;
$msg.=".\r\n";
return $msg;
}
@ -691,18 +708,25 @@ function get_stat($article) {
$msg="423 No article number selected\r\n";
return $msg;
}
$overviewfile=$workpath.$nntp_group."-overview";
$overviewfp=fopen($overviewfile, 'r');
while($overviewline=fgets($overviewfp)) {
$over=explode("\t", $overviewline);
if(trim($over[0]) == trim($article)) {
$msg="223 ".$article." ".$over[4]." status\r\n";
fclose($overviewfp);
return $msg;
}
$database = $spooldir.'/articles-overview.db3';
if(!is_file($database)) {
return false;
}
$dbh = rslight_db_open($database);
$query = $articles_dbh->prepare('SELECT * FROM overview WHERE number=:number AND newsgroup=:newsgroup');
$query->execute(['number' => $article, 'newsgroup' => $nntp_group]);
$found = 0;
while ($row = $query->fetch()) {
$found = 1;
break;
}
$dbh = null;
if($found == 1) {
$msg="223 ".$article." ".$row['msgid']." status\r\n";
} else {
$msg="423 No such article number ".$article."\r\n";
}
fclose($overviewfp);
$msg="423 No such article number ".$article."\r\n";
return $msg;
}
@ -1112,7 +1136,6 @@ $date_i,$mid_i,$references_i,$bytes_i,$lines_i,$xref_i,$body) {
touch($tmp_file, $article_date);
file_put_contents($logfile, "\n".format_log_date()." ".$section." Inserting local post: ".$nntp_group.":".$local, FILE_APPEND);
// Overview
$overviewHandle = fopen($spooldir."/".$nntp_group."-overview", 'a');
# Prepare overview database
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';

46
Rocksolid_Light/rslight/scripts/spoolnews.php
View File

@ -38,7 +38,7 @@ if(!isset($maxarticles_per_run)) {
$maxarticles_per_run = 100;
}
if(!isset($maxfirstrequest)) {
$maxfirstrequest = 1000;
$maxfirstrequest = 100;
}
if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) {
@ -198,30 +198,34 @@ function get_articles($ns, $group) {
if($article < $detail[2]) {
$article = $detail[2];
}
// Broken message on last run? Let's try again.
/*
if($article > ($detail[3])) {
$article = $detail[3];
}
*/
# Prepare databases
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$sql = 'INSERT OR IGNORE INTO '.$table.'(newsgroup, number, msgid, date, name, subject) VALUES(?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql);
// Articles Database
if($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3');
$article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql);
}
// Create list of message-ids
$group_overviewfile = $spooldir."/".$group."-overview";
$gover = file($group_overviewfile);
foreach($gover as $group_overview) {
$overview_msgid = explode("\t", $group_overview);
$msgids[trim($overview_msgid[4])] = true;
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup");
$stmt->bindParam(':newsgroup', $nntp_group);
$stmt->execute();
while($row = $stmt->fetch()) {
$msgids[$row['msgid']] = true;
break;
}
$dbh = null;
// Overview database
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$sql = 'INSERT OR IGNORE INTO overview(newsgroup, number, msgid, date, datestring, name, subject, refs, bytes, lines, xref) VALUES(?,?,?,?,?,?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql);
// Get overview from server
$server_overview = array();
$re = false;
@ -370,12 +374,8 @@ function get_articles($ns, $group) {
}
}
// Overview
$overviewHandle = fopen($workpath.$group."-overview", 'a');
fputs($overviewHandle, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n");
fclose($overviewHandle);
$stmt->execute([$group, $local, $mid[1], $article_date, $finddate[1], $from[1], $subject[1], $references, $bytes, $lines, $xref]);
$references="";
// add to database
$stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]);
if($CONFIG['article_database'] == '1') {
$this_article = file_get_contents($grouppath."/".$local);
// CREATE SEARCH SNIPPET