novabbs
/
rocksolid-light
2
0
Fork 1
Code Issues 15 Pull Requests Releases Wiki Activity

Add sqlite support for articles spool

This commit is contained in:
Retro_Guy 2021-01-20 15:54:47 -07:00
parent 12c7293708
commit 34ed3f90ce
9 changed files with 283 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Rocksolid_Light/rocksolid/lib/message.inc.php
View File

@ -467,10 +467,35 @@ function show_header($head,$group,$local_poster=false) {
function display_full_headers($id,$group,$name,$from) { function display_full_headers($id,$group,$name,$from) {
global $spoolpath, $CONFIG; global $spoolpath, $CONFIG;
$thisgroup = preg_replace('/\./', '/', $group); if($CONFIG['article_database'] == '1') {
$message = np_get_db_article($id, $group, 1);
foreach($message as $line) {
if(trim($line) == '') {
break;
}
if(stripos($line, 'Xref: ') === 0) {
continue;
}
if(stripos($line, 'From: ') === 0) {
$return.='From: ';
if(isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) {
$return.=truncate_email($from);
} else {
$return.=htmlspecialchars($from);
}
if ($name != "") {
$return.=' ('.htmlspecialchars($name).')';
}
$return.='<br />';
continue;
}
$return.=mb_decode_mimeheader(htmlspecialchars($line)).'<br />';
}
} else {
$thisgroup = preg_replace('/\./', '/', $group);
$message=fopen($spoolpath.$thisgroup.'/'.$id, 'r');
$message=fopen($spoolpath.$thisgroup.'/'.$id, 'r'); while($line=fgets($message)) {
while($line=fgets($message)) {
if(trim($line) == '') { if(trim($line) == '') {
break; break;
} }
@ -489,13 +514,11 @@ function display_full_headers($id,$group,$name,$from) {
} }
$return.='<br />'; $return.='<br />';
continue; continue;
} }
$return.=mb_decode_mimeheader(htmlspecialchars($line)).'<br />';
}
$return.=mb_decode_mimeheader(htmlspecialchars($line)).'<br />'; fclose($message);
} }
fclose($message);
return($return); return($return);
} }

1
Rocksolid_Light/rocksolid/lib/post.inc.php
View File

@ -261,6 +261,7 @@ function message_post($subject,$from,$newsgroups,$ref,$body,$encryptthis,$encryp
} }
if (isset($CONFIG['organization'])) if (isset($CONFIG['organization']))
fputs($ns,'Organization: '.quoted_printable_encode($CONFIG['organization'])."\r\n"); fputs($ns,'Organization: '.quoted_printable_encode($CONFIG['organization'])."\r\n");
$body=trim($body);
if ((isset($CONFIG['postfooter'])) && ($CONFIG['postfooter']!="")) { if ((isset($CONFIG['postfooter'])) && ($CONFIG['postfooter']!="")) {
$body.="\n-- \n".$CONFIG['postfooter']."\n".$_SERVER['HTTP_HOST']; $body.="\n-- \n".$CONFIG['postfooter']."\n".$_SERVER['HTTP_HOST'];
} }

72
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -1198,14 +1198,14 @@ function get_date_interval($value) {
return $variance; return $variance;
} }
function rslight_db_open($database, $table) { function rslight_db_open($database, $table='overview') {
try { try {
$dbh = new PDO('sqlite:'.$database); $dbh = new PDO('sqlite:'.$database);
} catch (PDOExeption $e) { } catch (PDOExeption $e) {
echo 'Connection failed: '.$e->getMessage(); echo 'Connection failed: '.$e->getMessage();
exit; exit;
} }
$dbh->exec("CREATE TABLE IF NOT EXISTS $table( $dbh->exec("CREATE TABLE IF NOT EXISTS overview(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
newsgroup TEXT, newsgroup TEXT,
number TEXT, number TEXT,
@ -1213,8 +1213,76 @@ function rslight_db_open($database, $table) {
date TEXT, date TEXT,
name TEXT, name TEXT,
subject TEXT)"); subject TEXT)");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_date on overview(date)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup on overview(newsgroup)');
$stmt->execute();
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS id_newsgroup_number on overview(newsgroup,number)');
$stmt->execute();
return($dbh); return($dbh);
} }
function article_db_open($database) {
try {
$dbh = new PDO('sqlite:'.$database);
} catch (PDOExeption $e) {
echo 'Connection failed: '.$e->getMessage();
exit;
}
$dbh->exec("CREATE TABLE IF NOT EXISTS articles(
id INTEGER PRIMARY KEY,
newsgroup TEXT,
number TEXT,
msgid TEXT,
date TEXT,
name TEXT,
subject TEXT,
article TEXT)");
$stmt = $dbh->query('CREATE INDEX IF NOT EXISTS db_number on articles(number)');
$stmt->execute();
return($dbh);
}
function np_get_db_article($article, $group, $makearray=1, $dbh=null) {
global $config_dir,$path,$groupconfig,$config_name,$logdir,$spooldir;
$logfile=$logdir.'/newsportal.log';
$msg2="";
$closeme = 0;
$database = $spooldir.'/'.$group.'-articles.db3';
if(!$dbh) {
$dbh = article_db_open($database);
$closeme = 1;
}
// By Message-ID
if(!is_numeric($article)) {
$stmt = $dbh->prepare("SELECT * FROM articles WHERE msgid like :terms");
$stmt->bindParam(':terms', $article);
$stmt->execute();
while($found = $stmt->fetch()) {
$msg2 = $found['article'];
break;
}
} else {
$stmt = $dbh->prepare("SELECT * FROM articles WHERE number = :terms");
$stmt->bindParam(':terms', $article);
$stmt->execute();
while($found = $stmt->fetch()) {
$msg2 = $found['article'];
break;
}
}
if($closeme == 1) {
$dbh = null;
}
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." DEBUG: fetched: ".$article." from ".$group, FILE_APPEND);
if($makearray == 1) {
$thisarticle = preg_split("/\r\n|\n|\r/", trim($msg2));
return $thisarticle;
} else {
return trim($msg2);
}
}
function get_config_value($configfile,$request) { function get_config_value($configfile,$request) {
global $config_dir; global $config_dir;

38
Rocksolid_Light/rocksolid/overboard.php
View File

@ -98,6 +98,7 @@ $table = 'overview';
$dbh = rslight_db_open($database, $table); $dbh = rslight_db_open($database, $table);
$query = $dbh->prepare('SELECT * FROM '.$table.' WHERE newsgroup=:findgroup AND date >= '.$oldest.' ORDER BY date DESC LIMIT '.$maxdisplay); $query = $dbh->prepare('SELECT * FROM '.$table.' WHERE newsgroup=:findgroup AND date >= '.$oldest.' ORDER BY date DESC LIMIT '.$maxdisplay);
$articles = array(); $articles = array();
$db_articles = array();
foreach($grouplist as $findgroup) { foreach($grouplist as $findgroup) {
$groups = preg_split("/(\ |\t)/", $findgroup, 2); $groups = preg_split("/(\ |\t)/", $findgroup, 2);
$findgroup = $groups[0]; $findgroup = $groups[0];
@ -108,16 +109,20 @@ foreach($grouplist as $findgroup) {
continue 2; continue 2;
} }
} }
/*
$thisgroup = preg_replace('/\./', '/', $findgroup); $thisgroup = preg_replace('/\./', '/', $findgroup);
if (!is_dir($spoolpath.$thisgroup)) { if (!is_dir($spoolpath.$thisgroup)) {
continue; continue;
} }
$stats = stat($spoolpath.$thisgroup); $stats = stat($spoolpath.$thisgroup);
if($stats[9] > $oldest) { if($stats[9] > $oldest) {
*/
if(1) {
if($dbh) { if($dbh) {
$query->execute(['findgroup' => $findgroup]); $query->execute(['findgroup' => $findgroup]);
while (($overviewline = $query->fetch()) !== false) { while (($overviewline = $query->fetch()) !== false) {
$articles[] = $spoolpath.$thisgroup.'/'.$overviewline['number']; $articles[] = $spoolpath.$thisgroup.'/'.$overviewline['number'];
$db_articles[] = $findgroup.':'.$overviewline['number'].':'.$overviewline['date'];
} }
} }
} }
@ -177,17 +182,29 @@ if (isset($_GET['thisgroup'])) {
$results=0; $results=0;
$files = array(); $files = array();
foreach($articles as $article) { if($CONFIG['article_database'] == '1') {
foreach($db_articles as $article) {
$order=explode(':', $article);
$files[$order[2]] = $article;
}
} else {
foreach($articles as $article) {
if(is_dir($article)) { if(is_dir($article)) {
continue; continue;
} }
$files[filemtime($article)] = $article; $files[filemtime($article)] = $article;
}
} }
krsort($files); krsort($files);
echo '<table cellspacing="0" width="100%" class="np_results_table">'; echo '<table cellspacing="0" width="100%" class="np_results_table">';
//date_default_timezone_set(timezone_name_from_abbr("", $CONFIG['timezone'] * 3600, 0)); //date_default_timezone_set(timezone_name_from_abbr("", $CONFIG['timezone'] * 3600, 0));
foreach($files as $article) { foreach($files as $article) {
$articledata = file_get_contents($article); if($CONFIG['article_database'] == '1') {
$data = explode(':', $article);
$articledata = np_get_db_article($data[1], $data[0], 0);
} else {
$articledata = file_get_contents($article);
}
$bodystart = strpos($articledata, $localeol); $bodystart = strpos($articledata, $localeol);
$header = substr($articledata, 0, $bodystart); $header = substr($articledata, 0, $bodystart);
@ -201,12 +218,17 @@ foreach($files as $article) {
} }
# Find group name and article number # Find group name and article number
$group = preg_replace($spoolpath_regexp, '', $article); if($CONFIG['article_database'] == '1') {
$group = preg_replace('/\//', '.', $group); $group = $data[0];
$findme = strrpos($group, '.'); $articlenumber = $data[1];
$groupname = substr($group, 0, $findme); $groupname = $group;
$articlenumber = substr($group, $findme+1); } else {
$group = preg_replace($spoolpath_regexp, '', $article);
$group = preg_replace('/\//', '.', $group);
$findme = strrpos($group, '.');
$groupname = substr($group, 0, $findme);
$articlenumber = substr($group, $findme+1);
}
# Generate link # Generate link
$url = $thissite."/article-flat.php?id=".$articlenumber."&group="._rawurlencode($groupname)."#".$articlenumber; $url = $thissite."/article-flat.php?id=".$articlenumber."&group="._rawurlencode($groupname)."#".$articlenumber;
$groupurl = $thissite."/thread.php?group="._rawurlencode($groupname); $groupurl = $thissite."/thread.php?group="._rawurlencode($groupname);

2
Rocksolid_Light/rocksolid/tail.inc
View File

@ -1,6 +1,6 @@
<center> <center>
<font size="1em"> <font size="1em">
<i>rocksolid light</i> 0.6.7 <i>rocksolid light</i> 0.6.8
<br /> <br />
<a href="https://github.com/novabbs/rocksolid-light" target=_blank>clearnet</a> <a href="https://github.com/novabbs/rocksolid-light" target=_blank>clearnet</a>
<a href="http://rslight.i2p/getrslight" target=_blank>i2p</a> <a href="http://rslight.i2p/getrslight" target=_blank>i2p</a>

19
Rocksolid_Light/rslight/scripts/expire.php
View File

@ -19,11 +19,6 @@
$webserver_group=$CONFIG['webserver_user']; $webserver_group=$CONFIG['webserver_user'];
$logfile=$logdir.'/expire.log'; $logfile=$logdir.'/expire.log';
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$query = $dbh->prepare('DELETE FROM '.$table.' WHERE newsgroup=:newsgroup AND number=:number');
$grouplist = file($config_dir.'/'.$config_name.'/groups.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); $grouplist = file($config_dir.'/'.$config_name.'/groups.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach($grouplist as $groupline) { foreach($grouplist as $groupline) {
$expireme = 0; $expireme = 0;
@ -40,6 +35,18 @@
if($expireme < 1) { if($expireme < 1) {
continue; continue;
} }
$database = $spooldir.'/articles-overview.db3';
$dbh = rslight_db_open($database);
$query = $dbh->prepare('DELETE FROM overview WHERE newsgroup=:newsgroup AND date<:expireme');
$query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$dbh = null;
if($CONFIG['article_database'] == '1') {
$database = $spooldir.'/'.$group.'-articles.db3';
$articles_dbh = article_db_open($database);
$articles_query = $articles_dbh->prepare('DELETE FROM articles WHERE newsgroup=:newsgroup AND date<:expireme');
$articles_query->execute([':newsgroup' => $group, ':expireme' => $expireme]);
$articles_dbh = null;
}
$grouppath = preg_replace('/\./', '/', $group); $grouppath = preg_replace('/\./', '/', $group);
$this_overview=$spooldir.'/'.$group.'-overview'; $this_overview=$spooldir.'/'.$group.'-overview';
@ -54,7 +61,6 @@
echo "Expiring: ".$break[4]." IN: ".$group." #".$break[0]."\r\n"; echo "Expiring: ".$break[4]." IN: ".$group." #".$break[0]."\r\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Expiring: ".$break[4]." IN: ".$group." #".$break[0], FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Expiring: ".$break[4]." IN: ".$group." #".$break[0], FILE_APPEND);
unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]); unlink($spooldir.'/articles/'.$grouppath.'/'.$break[0]);
$query->execute([':newsgroup' => $group, ':number' => $break[0]]);
continue; continue;
} else { } else {
fputs($out_overviewfp, $line); fputs($out_overviewfp, $line);
@ -66,7 +72,6 @@
chown($this_overview, $CONFIG['webserver_user']); chown($this_overview, $CONFIG['webserver_user']);
chgrp($this_overview, $webserver_group); chgrp($this_overview, $webserver_group);
} }
$dbh = null;
unlink($lockfile); unlink($lockfile);
touch($spooldir.'/'.$config_name.'-expire-timer'); touch($spooldir.'/'.$config_name.'-expire-timer');
?> ?>

13
Rocksolid_Light/rslight/scripts/nocem.php
View File

@ -99,13 +99,18 @@ function delete_message($messageid, $group) {
} }
if($config_name) { if($config_name) {
$database = $spooldir.'/articles-overview.db3'; $database = $spooldir.'/articles-overview.db3';
$table = 'overview'; $dbh = rslight_db_open($database);
$dbh = rslight_db_open($database, $table); $query = $dbh->prepare('DELETE FROM overview WHERE msgid=:messageid');
$query = $dbh->prepare('DELETE FROM '.$table.' WHERE msgid=:messageid');
$query->execute(['messageid' => $messageid]); $query->execute(['messageid' => $messageid]);
$dbh = null; $dbh = null;
} }
if($CONFIG['article_database'] == '1') {
$database = $spooldir.'/'.$group.'-articles.db3';
$articles_dbh = article_db_open($database);
$articles_query = $articles_dbh->prepare('DELETE FROM articles WHERE msgid=:messageid');
$articles_query->execute(['messageid' => $messageid]);
$articles_dbh = null;
}
$this_overview=$spooldir.'/'.$group.'-overview'; $this_overview=$spooldir.'/'.$group.'-overview';
if(false === (is_file($this_overview))) { if(false === (is_file($this_overview))) {
return; return;

130
Rocksolid_Light/rslight/scripts/rslight-lib.php
View File

@ -1,7 +1,7 @@
<?php <?php
function interact($msgsock, $use_crypto=false) function interact($msgsock, $use_crypto=false)
{ {
global $logdir,$logfile,$installed_path,$config_path,$groupconfig,$workpath,$path, $spooldir,$nntp_group,$nntp_article,$auth_ok,$user,$pass; global $CONFIG,$logdir,$logfile,$installed_path,$config_path,$groupconfig,$workpath,$path, $spooldir,$nntp_group,$nntp_article,$auth_ok,$user,$pass;
$workpath=$spooldir."/"; $workpath=$spooldir."/";
$path=$workpath."articles/"; $path=$workpath."articles/";
@ -474,10 +474,7 @@ function get_xhdr($header, $articles) {
$msg="221 Header information for ".$header." follows (from articles)\r\n"; $msg="221 Header information for ".$header." follows (from articles)\r\n";
for($i=$first; $i<=$last; $i++) { for($i=$first; $i<=$last; $i++) {
$article_full_path=$thisgroup.'/'.strval($i); $article_full_path=$thisgroup.'/'.strval($i);
if(!is_file($article_full_path)) { $data=extract_header_line($article_full_path, $header, $thisgroup, $i);
continue;
}
$data=extract_header_line($article_full_path, $header);
if($data !== false) { if($data !== false) {
if($mid !== false) { if($mid !== false) {
$msg.=$mid." ".$data; $msg.=$mid." ".$data;
@ -490,8 +487,13 @@ function get_xhdr($header, $articles) {
return $msg; return $msg;
} }
function extract_header_line($article_full_path, $header) { function extract_header_line($article_full_path, $header, $thisgroup, $article) {
$thisarticle=file($article_full_path, FILE_IGNORE_NEW_LINES); global $CONFIG;
if($CONFIG['article_database'] == '1') {
$thisarticle=get_db_article($article, $thisgroup);
} else {
$thisarticle=file($article_full_path, FILE_IGNORE_NEW_LINES);
}
foreach($thisarticle as $thisline) { foreach($thisarticle as $thisline) {
if($thisline == "") { if($thisline == "") {
$msg2.=".\r\n"; $msg2.=".\r\n";
@ -585,11 +587,6 @@ function get_stat($article) {
return $msg; return $msg;
} }
$overviewfile=$workpath.$nntp_group."-overview"; $overviewfile=$workpath.$nntp_group."-overview";
$thisgroup = $path."/".preg_replace('/\./', '/', $nntp_group);
if(!file_exists($thisgroup."/".$article)) {
$msg="423 No such article number ".$article."\r\n";
return $msg;
}
$overviewfp=fopen($overviewfile, 'r'); $overviewfp=fopen($overviewfile, 'r');
while($overviewline=fgets($overviewfp)) { while($overviewline=fgets($overviewfp)) {
$over=explode("\t", $overviewline); $over=explode("\t", $overviewline);
@ -605,13 +602,13 @@ function get_stat($article) {
} }
function get_article($article, $nntp_group) { function get_article($article, $nntp_group) {
global $config_dir,$path,$groupconfig,$config_name,$spooldir,$nntp_article; global $CONFIG,$config_dir,$path,$groupconfig,$config_name,$spooldir,$nntp_article;
$msg2=""; $msg2="";
// By Message-ID
// Use article pointer // Use article pointer
if(!isset($article) && is_numeric($nntp_article)) { if(!isset($article) && is_numeric($nntp_article)) {
$article = $nntp_article; $article = $nntp_article;
} }
// By Message-ID
if(!is_numeric($article)) { if(!is_numeric($article)) {
$database = $spooldir.'/articles-overview.db3'; $database = $spooldir.'/articles-overview.db3';
$table = 'overview'; $table = 'overview';
@ -622,6 +619,7 @@ function get_article($article, $nntp_group) {
while($found = $stmt->fetch()) { while($found = $stmt->fetch()) {
$nntp_group = $found['newsgroup']; $nntp_group = $found['newsgroup'];
$article = $found['number']; $article = $found['number'];
$this_id = $found['msgid'];
break; break;
} }
$dbh = null; $dbh = null;
@ -636,13 +634,17 @@ function get_article($article, $nntp_group) {
return $msg; return $msg;
} }
} }
if($CONFIG['article_database'] == '1') {
$thisarticle=get_db_article($article, $nntp_group);
} else {
$thisgroup = $path."/".preg_replace('/\./', '/', $nntp_group); $thisgroup = $path."/".preg_replace('/\./', '/', $nntp_group);
if(!file_exists($thisgroup."/".$article)) { if(!file_exists($thisgroup."/".$article)) {
$msg.="430 no such article found\r\n"; $msg.="430 no such article found\r\n";
return $msg; return $msg;
} }
$thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES); $thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES);
foreach($thisarticle as $thisline) { }
foreach($thisarticle as $thisline) {
if((strpos($thisline, "Message-ID: ") === 0) && !isset($mid[1])) { if((strpos($thisline, "Message-ID: ") === 0) && !isset($mid[1])) {
$mid=explode(': ', $thisline); $mid=explode(': ', $thisline);
} }
@ -653,45 +655,82 @@ function get_article($article, $nntp_group) {
return $msg.$msg2; return $msg.$msg2;
} }
function get_header($article, $nntp_group) { function get_db_article($article, $group) {
global $nntp_article,$config_dir,$path,$groupconfig,$config_name,$spooldir; global $nntp_article,$nntp_group,$config_dir,$path,$groupconfig,$config_name,$spooldir;
$msg2=""; $msg2="";
$database = $spooldir.'/'.$nntp_group.'-articles.db3';
$dbh = article_db_open($database);
// Use article pointer // Use article pointer
if(!isset($article) && is_numeric($nntp_article)) { if(!isset($article) && is_numeric($nntp_article)) {
$article = $nntp_article; $article = $nntp_article;
} }
// By Message-ID // By Message-ID
if(!is_numeric($article)) { if(!is_numeric($article)) {
$database = $spooldir.'/articles-overview.db3'; $stmt = $dbh->prepare("SELECT * FROM articles WHERE msgid like :terms");
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE msgid like :terms");
$stmt->bindParam(':terms', $article); $stmt->bindParam(':terms', $article);
$stmt->execute(); $stmt->execute();
while($found = $stmt->fetch()) { while($found = $stmt->fetch()) {
$nntp_group = $found['newsgroup']; $msg2 = $found['article'];
$article = $found['number'];
break; break;
} }
$dbh = null;
} else { } else {
// By article number $stmt = $dbh->prepare("SELECT * FROM articles WHERE number = :terms");
if($nntp_group === "") { $stmt->bindParam(':terms', $article);
$msg.="412 no newsgroup has been selected\r\n"; $stmt->execute();
return $msg; while($found = $stmt->fetch()) {
} $msg2 = $found['article'];
if(!is_numeric($article)) { break;
$msg.="420 no article has been selected\r\n";
return $msg;
} }
} }
$thisgroup = $path."/".preg_replace('/\./', '/', $nntp_group); $dbh = null;
if(!file_exists($thisgroup."/".$article)) {
$thisarticle = preg_split("/\r\n|\n|\r/", trim($msg2));
return $thisarticle;
}
function get_header($article, $nntp_group) {
global $CONFIG,$nntp_article,$config_dir,$path,$groupconfig,$config_name,$spooldir;
$msg2="";
// Use article pointer
if(!isset($article) && is_numeric($nntp_article)) {
$article = $nntp_article;
}
if($CONFIG['article_database'] == '1') {
$thisarticle=get_db_article($article, $nntp_group);
} else {
// By Message-ID
if(!is_numeric($article)) {
$database = $spooldir.'/articles-overview.db3';
$table = 'overview';
$dbh = rslight_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE msgid like :terms");
$stmt->bindParam(':terms', $article);
$stmt->execute();
while($found = $stmt->fetch()) {
$nntp_group = $found['newsgroup'];
$article = $found['number'];
break;
}
$dbh = null;
} else {
// By article number
if($nntp_group === "") {
$msg.="412 no newsgroup has been selected\r\n";
return $msg;
}
if(!is_numeric($article)) {
$msg.="420 no article has been selected\r\n";
return $msg;
}
}
$thisgroup = $path."/".preg_replace('/\./', '/', $nntp_group);
if(!file_exists($thisgroup."/".$article)) {
$msg.="430 no such article found\r\n"; $msg.="430 no such article found\r\n";
return $msg; return $msg;
}
$thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES);
} }
$thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES); foreach($thisarticle as $thisline) {
foreach($thisarticle as $thisline) {
if($thisline == "") { if($thisline == "") {
$msg2.=".\r\n"; $msg2.=".\r\n";
break; break;
@ -706,8 +745,15 @@ function get_header($article, $nntp_group) {
} }
function get_body($article, $nntp_group) { function get_body($article, $nntp_group) {
global $config_dir,$path,$groupconfig,$config_name,$spooldir; global $CONFIG,$nntp_article,$config_dir,$path,$groupconfig,$config_name,$spooldir;
$msg2=""; $msg2="";
// Use article pointer
if(!isset($article) && is_numeric($nntp_article)) {
$article = $nntp_article;
}
if($CONFIG['article_database'] == '1') {
$thisarticle=get_db_article($article, $nntp_group);
} else {
// By Message-ID // By Message-ID
if(!is_numeric($article)) { if(!is_numeric($article)) {
$menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); $menulist = file($config_dir."menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
@ -744,6 +790,7 @@ function get_body($article, $nntp_group) {
return $msg; return $msg;
} }
$thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES); $thisarticle=file($thisgroup."/".$article, FILE_IGNORE_NEW_LINES);
}
$body=0; $body=0;
foreach($thisarticle as $thisline) { foreach($thisarticle as $thisline) {
if(($thisline == "") && ($body == 0)) { if(($thisline == "") && ($body == 0)) {
@ -1015,6 +1062,15 @@ $date_i,$mid_i,$references_i,$bytes_i,$lines_i,$xref_i) {
$stmt = $dbh->prepare($sql); $stmt = $dbh->prepare($sql);
$stmt->execute([$nntp_group, $local, $mid_i, $article_date, $from_i, $subject_i]); $stmt->execute([$nntp_group, $local, $mid_i, $article_date, $from_i, $subject_i]);
$dbh = null; $dbh = null;
}
if($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir.'/'.$nntp_group.'-articles.db3');
$article_sql = 'INSERT INTO articles(newsgroup, number, msgid, date, name, subject, article) VALUES(?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql);
$this_article = file_get_contents($grouppath."/".$local);
$article_stmt->execute([$nntp_group, $local, $mid_i, $article_date, $from_i, $subject_i, trim($this_article)]);
unlink($grouppath."/".$local);
$article_dbh = null;
} }
fputs($overviewHandle, $local."\t".$subject_i."\t".$from_i."\t".$date_i."\t".$mid_i."\t".$references_i."\t".$bytes_i."\t".$lines_i."\t".$xref_i."\n"); fputs($overviewHandle, $local."\t".$subject_i."\t".$from_i."\t".$date_i."\t".$mid_i."\t".$references_i."\t".$bytes_i."\t".$lines_i."\t".$xref_i."\n");
fclose($overviewHandle); fclose($overviewHandle);

53
Rocksolid_Light/rslight/scripts/spoolnews.php
View File

@ -36,7 +36,7 @@ if(!isset($maxarticles_per_run)) {
$maxarticles_per_run = 100; $maxarticles_per_run = 100;
} }
if(!isset($maxfirstrequest)) { if(!isset($maxfirstrequest)) {
$maxfirstrequest = 1000; $maxfirstrequest = 50000;
} }
if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) { if(!isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) {
@ -118,13 +118,17 @@ echo "\nSpoolnews Done\r\n";
function get_articles($ns, $group) { function get_articles($ns, $group) {
global $enable_rslight, $spooldir, $CONFIG, $maxarticles_per_run, $maxfirstrequest, $workpath, $path, $remote_groupfile, $local_groupfile, $local, $logdir, $config_name, $logfile; global $enable_rslight, $spooldir, $CONFIG, $maxarticles_per_run, $maxfirstrequest, $workpath, $path, $remote_groupfile, $local_groupfile, $local, $logdir, $config_name, $logfile;
# Prepare search database (this is only for testing atm) # Prepare databases
$database = $spooldir.'/articles-overview.db3'; $database = $spooldir.'/articles-overview.db3';
$table = 'overview'; $table = 'overview';
$dbh = rslight_db_open($database, $table); $dbh = rslight_db_open($database, $table);
$sql = 'INSERT INTO '.$table.'(newsgroup, number, msgid, date, name, subject) VALUES(?,?,?,?,?,?)'; $sql = 'INSERT INTO '.$table.'(newsgroup, number, msgid, date, name, subject) VALUES(?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql); $stmt = $dbh->prepare($sql);
if($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir.'/'.$group.'-articles.db3');
$article_sql = 'INSERT INTO articles(newsgroup, number, msgid, date, name, subject, article) VALUES(?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql);
}
if($ns == false) { if($ns == false) {
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Lost connection to ".$CONFIG['remote_server'].":".$CONFIG['remote_port'], FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Lost connection to ".$CONFIG['remote_server'].":".$CONFIG['remote_port'], FILE_APPEND);
exit(); exit();
@ -158,15 +162,7 @@ function get_articles($ns, $group) {
// Try to find last article number in local_groupfile // Try to find last article number in local_groupfile
$local = get_high_watermark($group); $local = get_high_watermark($group);
if(!is_numeric($local)) { if(!is_numeric($local)) {
$thisgroup = $path."/".preg_replace('/\./', '/', $group); $ok_article = get_article_list($group);
$articles = scandir($thisgroup);
$ok_article=array();
foreach($articles as $this_article) {
if(!is_numeric($this_article)) {
continue;
}
$ok_article[]=$this_article;
}
sort($ok_article); sort($ok_article);
$local = $ok_article[key(array_slice($ok_article, -1, 1, true))]; $local = $ok_article[key(array_slice($ok_article, -1, 1, true))];
if(!is_numeric($local)) { if(!is_numeric($local)) {
@ -330,13 +326,17 @@ function get_articles($ns, $group) {
fputs($overviewHandle, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n"); fputs($overviewHandle, $local."\t".$subject[1]."\t".$from[1]."\t".$finddate[1]."\t".$mid[1]."\t".$references."\t".$bytes."\t".$lines."\t".$xref."\n");
fclose($overviewHandle); fclose($overviewHandle);
$references=""; $references="";
// add to search database // add to database
$stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]); $stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1]]);
// End Overview if($CONFIG['article_database'] == '1') {
$this_article = file_get_contents($grouppath."/".$local);
if($article_date > time()) $article_stmt->execute([$group, $local, $mid[1], $article_date, $from[1], $subject[1], $this_article]);
$article_date = time(); unlink($grouppath."/".$local);
touch($grouppath."/".$local, $article_date); } else {
if($article_date > time())
$article_date = time();
touch($grouppath."/".$local, $article_date);
}
echo "\nRetrieved: ".$group." ".$article."\n"; echo "\nRetrieved: ".$group." ".$article."\n";
file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Wrote to spool: ".$CONFIG['remote_server']." ".$group.":".$article, FILE_APPEND); file_put_contents($logfile, "\n".format_log_date()." ".$config_name." Wrote to spool: ".$CONFIG['remote_server']." ".$group.":".$article, FILE_APPEND);
$i++; $i++;
@ -386,6 +386,9 @@ function get_articles($ns, $group) {
} }
} }
fclose($saveconfig); fclose($saveconfig);
if($CONFIG['article_database'] == '1') {
$article_dbh = null;
}
$dbh = null; $dbh = null;
} }
@ -482,4 +485,18 @@ function get_high_watermark($group) {
} }
} }
function get_article_list($thisgroup) {
global $spooldir;
$group_overviewfp=fopen($spooldir."/".$thisgroup."-overview", 'r');
$ok_article=array();
while($line = fgets($group_overviewfp)) {
$art=explode("\t", $line);
if(is_numeric($art[0])) {
$ok_article[] = $art[0];
}
}
fclose($group_overviewfp);
return($ok_article);
}
?> ?>