From ed31b5b6a1a96d9ec1ebd9c44ea5000965e21cb8 Mon Sep 17 00:00:00 2001 From: Retro_Guy Date: Sun, 13 Aug 2023 05:08:04 -0700 Subject: [PATCH] Remove flat file overview and use articles-overview.db3 for all overview features. --- Rocksolid_Light/rocksolid/newsportal.php | 5 + Rocksolid_Light/rslight/scripts/expire.php | 36 ++---- Rocksolid_Light/rslight/scripts/nocem.php | 37 ++---- .../rslight/scripts/rslight-lib.php | 113 +++++++++++------- Rocksolid_Light/rslight/scripts/spoolnews.php | 46 +++---- 5 files changed, 118 insertions(+), 119 deletions(-) diff --git a/Rocksolid_Light/rocksolid/newsportal.php b/Rocksolid_Light/rocksolid/newsportal.php index f89ed73..128b146 100644 --- a/Rocksolid_Light/rocksolid/newsportal.php +++ b/Rocksolid_Light/rocksolid/newsportal.php @@ -1462,8 +1462,13 @@ function rslight_db_open($database, $table='overview') { number TEXT, msgid TEXT, date TEXT, + datestring TEXT, name TEXT, subject TEXT, + refs TEXT, + bytes TEXT, + lines TEXT, + xref TEXT, unique (newsgroup, msgid))"); $stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)'); $stmt->execute(); diff --git a/Rocksolid_Light/rslight/scripts/expire.php b/Rocksolid_Light/rslight/scripts/expire.php index 26be9ad..2e3d0ff 100755 --- a/Rocksolid_Light/rslight/scripts/expire.php +++ b/Rocksolid_Light/rslight/scripts/expire.php @@ -61,35 +61,17 @@ file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." $articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]); $articles_dbh = null; } - } - -echo "Expiring group overview file...\n"; -file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring group overview file...", FILE_APPEND); - $grouppath = preg_replace('/\./', '/', $group); - $this_overview=$spooldir.'/'.$group.'-overview'; - $out_overview=$this_overview.'.new'; - $overviewfp=fopen($this_overview, 'r'); - $out_overviewfp=fopen($out_overview, 'w'); - while($line=fgets($overviewfp)) { - $break=explode("\t", $line); - if(strtotime($break[3]) < $expireme) { - echo "Expiring: ".$break[4]." IN: ".$group." #".$break[0]."\r\n"; - file_put_contents($logfile, "\n".format_log_date()." ".$config_name." ".$group." Expiring: ".$break[4], FILE_APPEND); - // Remove article from tradspool: - if(is_file($spooldir.'/articles/'.$grouppath.'/'.$break[0])) { - unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]); + } else { // Expire tradspool + $database = $spooldir.'/articles-overview.db3'; + $dbh = rslight_db_open($database); + $query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND date<:expireme'); + $query->execute([':newsgroup' => $group, ':expireme' => $expireme]); + $grouppath = preg_replace('/\./', '/', $group); + while($row = $query->fetch()) { + unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']); } - thread_cache_removearticle($group,$break[4]); - continue; - } else { - fputs($out_overviewfp, $line); - } + $dbh = null; } - fclose($overviewfp); - fclose($out_overviewfp); - rename($out_overview, $this_overview); - chown($this_overview, $CONFIG['webserver_user']); - chgrp($this_overview, $webserver_group); } unlink($lockfile); touch($spooldir.'/'.$config_name.'-expire-timer'); diff --git a/Rocksolid_Light/rslight/scripts/nocem.php b/Rocksolid_Light/rslight/scripts/nocem.php index 10bbefc..edcfb56 100755 --- a/Rocksolid_Light/rslight/scripts/nocem.php +++ b/Rocksolid_Light/rslight/scripts/nocem.php @@ -107,30 +107,19 @@ function delete_message($messageid, $group) { $articles_dbh = null; } } - $this_overview=$spooldir.'/'.$group.'-overview'; - if(false === (is_file($this_overview))) { - return; - } - $out_overview=$this_overview.'.new'; - $overviewfp=fopen($this_overview, 'r'); - $out_overviewfp=fopen($out_overview, 'w'); - while($line=fgets($overviewfp)) { - $break=explode("\t", $line); - if($break[4] == $messageid) { - echo "DELETING: ".$messageid." IN: ".$group." #".$break[0]."\r\n"; - file_put_contents($logfile, "\n".format_log_date()." ".$config_name." DELETING: ".$messageid." IN: ".$group." #".$break[0], FILE_APPEND); - $grouppath = preg_replace('/\./', '/', $group); - unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]); - continue; - } else { - fputs($out_overviewfp, $line); - } - } - fclose($overviewfp); - fclose($out_overviewfp); - rename($out_overview, $this_overview); - chown($this_overview, $CONFIG['webserver_user']); - chgrp($this_overview, $webserver_group); + +// Tradspool + if($CONFIG['article_database'] != '1') { + $database = $spooldir.'/articles-overview.db3'; + $dbh = rslight_db_open($database); + $query = $dbh->prepare('SELECT FROM overview WHERE newsgroup=:newsgroup AND msgid<:msgid'); + $query->execute([':newsgroup' => $group, ':msgid' => $messageid]); + $grouppath = preg_replace('/\./', '/', $group); + while($row = $query->fetch()) { + unlink($spooldir.'/articles/'.$grouppath.'/'.$row['number']); + } + $dbh = null; + } delete_message_from_overboard($config_name, $group, $messageid); return; } diff --git a/Rocksolid_Light/rslight/scripts/rslight-lib.php b/Rocksolid_Light/rslight/scripts/rslight-lib.php index de9fba4..5e675c3 100755 --- a/Rocksolid_Light/rslight/scripts/rslight-lib.php +++ b/Rocksolid_Light/rslight/scripts/rslight-lib.php @@ -252,11 +252,14 @@ set_time_limit(0); } function prepare_post($filename) { - global $logdir; + global $logdir, $spooldir; $logfile = $logdir.'/nntp.log'; $message = file($filename, FILE_IGNORE_NEW_LINES); $lines = 0; $is_header = 1; + $nocem_check="@@NCM"; + $bbsmail_check="@@RSL"; + foreach($message as $line) { if(trim($line) == "" || $lines > 0) { $is_header=0; @@ -269,7 +272,13 @@ function prepare_post($filename) { $ngroups=explode(': ', $line); $newsgroups=$ngroups[1]; $lines++; - break; + continue; + } + if(stripos($line, "Subject: ") === 0) { + $sub=explode(': ', $line); + $subject=$sub[1]; + $lines++; + continue; } } $ngroups = preg_split("/(\ |\,)/", trim($newsgroups)); @@ -283,6 +292,17 @@ function prepare_post($filename) { } } if($ok == 1) { + if((strpos($rslight_gpg['nntp_group'], $group) !== false) && ($rslight_gpg['enable'] == '1')) { + if(strpos($subject, $bbsmail_check) !== false) { + $bbsmail_file = preg_replace('/@@RSL /', '', $subject); + $bbsmail_filename = $spooldir."/bbsmail/in/bbsmail-".$bbsmail_file; + copy($filename, $bbsmail_filename); + } + } + if(strpos($subject, $nocem_check) !== false) { + $nocem_file = tempnam($spooldir."/nocem", "nocem-".$group."-"); + copy($filename, $nocem_file); + } $response="240 Article received OK\r\n"; } else { $response="441 Posting failed\r\n"; @@ -433,25 +453,25 @@ function process_post($message, $group) { if($response == "") { // Check for duplicate msgid $duplicate=0; - if(file_exists($spooldir."/".$group."-overview")) { - $group_overviewfp=fopen($spooldir."/".$group."-overview", 'r'); - while($group_overview=fgets($group_overviewfp, 2048)) { - $overview_msgid = explode("\t", $group_overview); - if(strpos($overview_msgid[4], $msgid) !== false) { - unlink($postfilename); + $database = $spooldir.'/articles-overview.db3'; + $table = 'overview'; + $dbh = rslight_db_open($database, $table); + $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND msgid=:msgid ORDER BY number"); + $stmt->execute(['thisgroup' => $group, ':msgid' => $msgid]); + while($found = $stmt->fetch()) { + unlink($postfilename); file_put_contents($logfile, "\n".format_log_date()." ".$section." Duplicate Message-ID for: ".$msgid, FILE_APPEND); - $duplicate=1; - break; - } - } - fclose($group_overviewfp); - } - if($duplicate == 0) { - $response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body); - } else { - $response="441 Posting failed (duplicate)\r\n"; + $duplicate=1; + break; + } + $dbh = null; + + if($duplicate == 0) { + $response = insert_article($section,$group,$postfilename,$subject[1],$from[1],$article_date,$date_rep,$msgid,$references,$bytes,$lines,$xref,$body); + } else { + $response="441 Posting failed (duplicate)\r\n"; + } } - } } return $response; } @@ -610,7 +630,7 @@ function get_title($mode) { } function get_xover($articles, $msgsock) { - global $nntp_group,$nntp_article,$workpath,$path; + global $nntp_group,$nntp_article,$workpath,$path,$spooldir; // Use article pointer if(!isset($articles) && is_numeric($nntp_article)) { $articles = $nntp_article; @@ -637,7 +657,6 @@ function get_xover($articles, $msgsock) { $msg="420 no article(s) selected\r\n"; return $msg; } - $overviewfile=$workpath.$nntp_group."-overview"; if(!isset($this_id)) { $article_num = explode('-', $articles); $first = $article_num[0]; @@ -660,19 +679,17 @@ function get_xover($articles, $msgsock) { } } fwrite($msgsock, $output, strlen($output)); - if(file_exists($overviewfile)) { - $overviewfp=fopen($overviewfile, 'r'); - while($overviewline=fgets($overviewfp)) { - $article=preg_split("/[\s,]+/", $overviewline); - for($i=$first; $i<=$last; $i++) { - if($article[0] === strval($i)) { - $overviewline = trim($overviewline)."\r\n"; - fwrite($msgsock, $overviewline, strlen($overviewline)); - } - } - } - fclose($overviewfp); + + $database = $spooldir.'/articles-overview.db3'; + $table = 'overview'; + $dbh = rslight_db_open($database, $table); + $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup AND number BETWEEN :first AND :last ORDER BY number"); + $stmt->execute(['thisgroup' => $nntp_group, ':first' => $first, ':last' => $last]); + $msg = ''; + while($row = $stmt->fetch()) { + $msg.= $row['number']."\t".$row['subject']."\t".$row['name']."\t".$row['datestring']."\t".$row['msgid']."\t".$row['refs']."\t".$row['bytes']."\t".$row['lines']."\t".$row['xref']."\r\n"; } + $dbh = null; $msg.=".\r\n"; return $msg; } @@ -691,18 +708,25 @@ function get_stat($article) { $msg="423 No article number selected\r\n"; return $msg; } - $overviewfile=$workpath.$nntp_group."-overview"; - $overviewfp=fopen($overviewfile, 'r'); - while($overviewline=fgets($overviewfp)) { - $over=explode("\t", $overviewline); - if(trim($over[0]) == trim($article)) { - $msg="223 ".$article." ".$over[4]." status\r\n"; - fclose($overviewfp); - return $msg; - } + + $database = $spooldir.'/articles-overview.db3'; + if(!is_file($database)) { + return false; + } + $dbh = rslight_db_open($database); + $query = $articles_dbh->prepare('SELECT * FROM overview WHERE number=:number AND newsgroup=:newsgroup'); + $query->execute(['number' => $article, 'newsgroup' => $nntp_group]); + $found = 0; + while ($row = $query->fetch()) { + $found = 1; + break; + } + $dbh = null; + if($found == 1) { + $msg="223 ".$article." ".$row['msgid']." status\r\n"; + } else { + $msg="423 No such article number ".$article."\r\n"; } - fclose($overviewfp); - $msg="423 No such article number ".$article."\r\n"; return $msg; } @@ -1112,7 +1136,6 @@ $date_i,$mid_i,$references_i,$bytes_i,$lines_i,$xref_i,$body) { touch($tmp_file, $article_date); file_put_contents($logfile, "\n".format_log_date()." ".$section." Inserting local post: ".$nntp_group.":".$local, FILE_APPEND); // Overview - $overviewHandle = fopen($spooldir."/".$nntp_group."-overview", 'a'); # Prepare overview database $database = $spooldir.'/articles-overview.db3'; $table = 'overview'; diff --git a/Rocksolid_Light/rslight/scripts/spoolnews.php b/Rocksolid_Light/rslight/scripts/spoolnews.php index 47202f3..f39291a 100755 --- a/Rocksolid_Light/rslight/scripts/spoolnews.php +++ b/Rocksolid_Light/rslight/scripts/spoolnews.php @@ -38,7 +38,7 @@ if(!isset($maxarticles_per_run)) { $maxarticles_per_run = 100; } if(!isset($maxfirstrequest)) { - $maxfirstrequest = 1000; + $maxfirstrequest = 100; } if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) { @@ -198,30 +198,34 @@ function get_articles($ns, $group) { if($article < $detail[2]) { $article = $detail[2]; } -// Broken message on last run? Let's try again. -/* - if($article > ($detail[3])) { - $article = $detail[3]; - } -*/ -# Prepare databases - $database = $spooldir.'/articles-overview.db3'; - $table = 'overview'; - $dbh = rslight_db_open($database, $table); - $sql = 'INSERT OR IGNORE INTO '.$table.'(newsgroup, number, msgid, date, name, subject) VALUES(?,?,?,?,?,?)'; - $stmt = $dbh->prepare($sql); + +// Articles Database if($CONFIG['article_database'] == '1') { $article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3'); $article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)'; $article_stmt = $article_dbh->prepare($article_sql); } + // Create list of message-ids - $group_overviewfile = $spooldir."/".$group."-overview"; - $gover = file($group_overviewfile); - foreach($gover as $group_overview) { - $overview_msgid = explode("\t", $group_overview); - $msgids[trim($overview_msgid[4])] = true; + $database = $spooldir.'/articles-overview.db3'; + $table = 'overview'; + $dbh = rslight_db_open($database, $table); + $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup"); + $stmt->bindParam(':newsgroup', $nntp_group); + $stmt->execute(); + while($row = $stmt->fetch()) { + $msgids[$row['msgid']] = true; + break; } + $dbh = null; + +// Overview database + $database = $spooldir.'/articles-overview.db3'; + $table = 'overview'; + $dbh = rslight_db_open($database, $table); + $sql = 'INSERT OR IGNORE INTO overview(newsgroup, number, msgid, date, datestring, name, subject, refs, bytes, lines, xref) VALUES(?,?,?,?,?,?,?,?,?,?,?)'; + $stmt = $dbh->prepare($sql); + // Get overview from server $server_overview = array(); $re = false; @@ -370,12 +374,8 @@ function get_articles($ns, $group) { } } // Overview - $overviewHandle = fopen($workpath.$group."-overview", 'a'); - fputs($overviewHandle, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n"); - fclose($overviewHandle); + $stmt->execute([$group, $local, $mid[1], $article_date, $finddate[1], $from[1], $subject[1], $references, $bytes, $lines, $xref]); $references=""; -// add to database - $stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]); if($CONFIG['article_database'] == '1') { $this_article = file_get_contents($grouppath."/".$local); // CREATE SEARCH SNIPPET