novabbs
/
rocksolid-light
3
0
Fork 1
Code Issues 25 Pull Requests Releases Wiki Activity

Create more standard Xref for incoming articles for better newsreader support.

This commit is contained in:
Retro_Guy 2023-10-23 19:01:30 -07:00
parent 25bba4e9c1
commit 8f5ea878de
3 changed files with 184 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Rocksolid_Light/rocksolid/lib/message.inc.php
View File

@ -589,9 +589,6 @@ function display_full_headers($article, $group, $name, $from, $getface = false)
if (stripos($line, ': ') !== false) { if (stripos($line, ': ') !== false) {
$isface = 0; $isface = 0;
} }
if (stripos($line, 'Xref: ') === 0) {
continue;
}
if (stripos($line, 'From: ') === 0) { if (stripos($line, 'From: ') === 0) {
$return .= 'From: '; $return .= 'From: ';
if (isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) { if (isset($CONFIG['hide_email']) && $CONFIG['hide_email'] == true) {

116
Rocksolid_Light/rocksolid/newsportal.php
View File

@ -123,6 +123,7 @@ function nntp2_open($nserver = 0, $nport = 0)
} }
} }
// $ns=@fsockopen($nserver,$nport); // $ns=@fsockopen($nserver,$nport);
// echo "PORT: ".$nport." ns: ".$ns;
$weg = line_read($ns); // kill the first line $weg = line_read($ns); // kill the first line
if (substr($weg, 0, 2) != "20") { if (substr($weg, 0, 2) != "20") {
echo "<p>" . $text_error["error:"] . $weg . "</p>"; echo "<p>" . $text_error["error:"] . $weg . "</p>";
@ -1860,15 +1861,126 @@ function verify_gpg_signature($res, $signed_text)
} }
} }
function get_next_article_number($group)
{
$ok_article = get_article_list($group);
sort($ok_article);
$local = $ok_article[key(array_slice($ok_article, - 1, 1, true))];
if (! is_numeric($local)) {
$local = 0;
}
$local = $local + 1;
if ($local < 1) {
$local = 1;
}
while (is_deleted_post($group, $local)) {
$local ++;
}
return $local;
}
function insert_article_from_array($this_article)
{
global $CONFIG, $config_name, $spooldir, $logdir;
$logfile = $logdir . '/spoolnews.log';
$group = $this_article['group'];
$grouppath = $path . preg_replace('/\./', '/', $group);
// Create list of message-ids
$database = $spooldir . '/articles-overview.db3';
$table = 'overview';
$dbh = overview_db_open($database, $table);
$stmt = $dbh->prepare("SELECT msgid FROM $table WHERE newsgroup=:newsgroup");
$stmt->bindParam(':newsgroup', $group);
$stmt->execute();
while ($row = $stmt->fetch()) {
$msgids[$row['msgid']] = true;
}
$dbh = null;
// Check history database for deleted message-ids
$database = $spooldir . '/history.db3';
$table = 'history';
$dbh = history_db_open($database, $table);
$stmt = $dbh->prepare("SELECT msgid FROM $table WHERE newsgroup=:newsgroup");
$stmt->bindParam(':newsgroup', $group);
$stmt->execute();
while ($row = $stmt->fetch()) {
$msgids[$row['msgid']] = true;
}
$dbh = null;
if ($msgids[$this_article['mid']] == true) {
echo "\nDuplicate Message-ID for: " . $group . ":" . $this_article['local'];
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Duplicate Message-ID for: " . $group . ":" . $this_article['local'], FILE_APPEND);
return "441 Insert failed (duplicate)\r\n";
}
// Open articles Database
if ($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir . '/' . $group . '-articles.db3');
$article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql);
}
// Open overview database
$database = $spooldir . '/articles-overview.db3';
$table = 'overview';
$overview_dbh = overview_db_open($database, $table);
$overview_sql = 'INSERT OR IGNORE INTO overview(newsgroup, number, msgid, date, datestring, name, subject, refs, bytes, lines, xref) VALUES(?,?,?,?,?,?,?,?,?,?,?)';
$overview_stmt = $overview_dbh->prepare($overview_sql);
// Overview
$overview_stmt->execute([
$group,
$this_article['local'],
$this_article['mid'],
$this_article['epochdate'],
$this_article['stringdate'],
$this_article['from'],
$this_article['subject'],
$this_article['references'],
$this_article['bytes'],
$this_article['lines'],
$this_article['xref']
]);
$overview_dbh = null;
$references = "";
// Articles
if ($CONFIG['article_database'] == '1') {
$article_stmt->execute([
$group,
$this_article['local'],
$this_article['mid'],
$this_article['epochdate'],
$this_article['from'],
$this_article['subject'],
$this_article['article'],
$this_article['snippet']
]);
unlink($grouppath . "/" . $this_article['local']);
$article_dbh = null;
} else {
if ($article_date > time())
$article_date = time();
touch($grouppath . "/" . $this_article['local'], $article_date);
}
echo "\nRetrieved: " . $group . " " . $this_article['local'];
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Spooling: " . $group . ":" . $this_article['local'], FILE_APPEND);
$status = "spooled";
$statusdate = time();
$statusreason = "imported";
add_to_history($group, $this_article['local'], $this_article['mid'], $status, $statusdate, $statusreason, $statusnotes);
}
function is_deleted_post($group, $number) function is_deleted_post($group, $number)
{ {
global $spooldir; global $spooldir;
$database = $spooldir . '/history.db3'; $database = $spooldir . '/history.db3';
$table = 'history'; $table = 'history';
$dbh = history_db_open($database, $table); $dbh = history_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup AND number=:nicole"); $stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:newsgroup AND number=:newsnum");
$stmt->bindParam(':newsgroup', $group); $stmt->bindParam(':newsgroup', $group);
$stmt->bindParam(':nicole', $number); $stmt->bindParam(':newsnum', $number);
$stmt->execute(); $stmt->execute();
$status = false; $status = false;
while ($row = $stmt->fetch()) { while ($row = $stmt->fetch()) {

130
Rocksolid_Light/rslight/scripts/spoolnews.php
View File

@ -113,6 +113,7 @@ if ($CONFIG['remote_server'] != '') {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " remote_server: " . $CONFIG['remote_server'], FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " remote_server: " . $CONFIG['remote_server'], FILE_APPEND);
$ns = nntp2_open($CONFIG['remote_server'], $CONFIG['remote_port']); $ns = nntp2_open($CONFIG['remote_server'], $CONFIG['remote_port']);
$ns2 = nntp_open(); $ns2 = nntp_open();
echo 'Open ns2: ' . $ns2 . "\n";
if (! $ns) { if (! $ns) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Failed to connect to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'], FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Failed to connect to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'], FILE_APPEND);
exit(); exit();
@ -130,6 +131,7 @@ if ($CONFIG['remote_server'] != '') {
if ($enable_rslight == 1) { if ($enable_rslight == 1) {
if ($timer) { if ($timer) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Updating threads for: " . $name[0] . "...", FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Updating threads for: " . $name[0] . "...", FILE_APPEND);
echo 'Use ns2: ' . $ns2 . "\n";
thread_load_newsserver($ns2, $name[0], 0); thread_load_newsserver($ns2, $name[0], 0);
} }
} }
@ -182,21 +184,8 @@ function get_articles($ns, $group)
} }
} }
if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) { if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) {
// Get next available article number for group
// Get list of article numbers to find what number is next $local = get_next_article_number($group);
$ok_article = get_article_list($group);
sort($ok_article);
$local = $ok_article[key(array_slice($ok_article, - 1, 1, true))];
if (! is_numeric($local)) {
$local = 0;
}
$local = $local + 1;
if ($local < 1) {
$local = 1;
}
while (is_deleted_post($group, $local)) {
$local ++;
}
} }
# Split group response line to get last article number # Split group response line to get last article number
$detail = explode(" ", $response); $detail = explode(" ", $response);
@ -210,12 +199,6 @@ function get_articles($ns, $group)
$article = $detail[2]; $article = $detail[2];
} }
// Articles Database
if ($CONFIG['article_database'] == '1') {
$article_dbh = article_db_open($spooldir . '/' . $group . '-articles.db3');
$article_sql = 'INSERT OR IGNORE INTO articles(newsgroup, number, msgid, date, name, subject, article, search_snippet) VALUES(?,?,?,?,?,?,?,?)';
$article_stmt = $article_dbh->prepare($article_sql);
}
// Create list of message-ids // Create list of message-ids
$database = $spooldir . '/articles-overview.db3'; $database = $spooldir . '/articles-overview.db3';
$table = 'overview'; $table = 'overview';
@ -239,12 +222,6 @@ function get_articles($ns, $group)
$msgids[$row['msgid']] = true; $msgids[$row['msgid']] = true;
} }
$dbh = null; $dbh = null;
// Overview database
$database = $spooldir . '/articles-overview.db3';
$table = 'overview';
$dbh = overview_db_open($database, $table);
$sql = 'INSERT OR IGNORE INTO overview(newsgroup, number, msgid, date, datestring, name, subject, refs, bytes, lines, xref) VALUES(?,?,?,?,?,?,?,?,?,?,?)';
$stmt = $dbh->prepare($sql);
// Get overview from server // Get overview from server
$server_overview = array(); $server_overview = array();
@ -268,16 +245,22 @@ function get_articles($ns, $group)
# Pull articles and save them in our spool # Pull articles and save them in our spool
@mkdir($grouppath, 0755, 'recursive'); @mkdir($grouppath, 0755, 'recursive');
$i = 0; $i = 0;
// GET INDIVIDUAL ARTICLE
while ($article <= $detail[3]) { while ($article <= $detail[3]) {
if (! is_numeric($article)) { if (! is_numeric($article)) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " DEBUG This should show server group:article number: " . $CONFIG['remote_server'] . " " . $group . ":" . $article, FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " DEBUG This should show server group:article number: " . $CONFIG['remote_server'] . " " . $group . ":" . $article, FILE_APPEND);
break; break;
} }
// Create array for article, then send to insert_article_from_array()
if (isset($current_article)) {
unset($current_article);
$current_article = array();
}
if ($CONFIG['enable_nntp'] != true) { if ($CONFIG['enable_nntp'] != true) {
$local = $article; $local = $article;
} }
if ($msgids[$overview_msgid[$article]] == true) { if ($msgids[$overview_msgid[$article]] == true) {
echo "\nDuplicate Message-ID for: " . $group . ":" . $article; echo "\nDuplicate Message-ID for: " . $group . ":" . $local;
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Duplicate Message-ID for: " . $group . ":" . $article, FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Duplicate Message-ID for: " . $group . ":" . $article, FILE_APPEND);
$article ++; $article ++;
continue; continue;
@ -299,8 +282,28 @@ function get_articles($ns, $group)
$is_header = 1; $is_header = 1;
$body = ""; $body = "";
while (strcmp($response, ".") != 0) { while (strcmp($response, ".") != 0) {
$is_xref = false;
$bytes = $bytes + mb_strlen($response, '8bit'); $bytes = $bytes + mb_strlen($response, '8bit');
if (trim($response) == "" || $lines > 0) { if (trim($response) == "" || $lines > 0) {
// Create Xref: header
$current_article['xref'] = "Xref: " . $CONFIG['pathhost'];
foreach ($allgroups as $agroup) {
$agroup = trim($agroup);
if ((! testGroup($agroup)) || $agroup == '') {
continue;
}
if ($group == $agroup) {
$artnum = $local;
} else {
$artnum = get_next_article_number($agroup);
}
if($artnum > 0) {
$current_article['xref'] .= ' ' . $agroup . ':' . $artnum;
}
}
if ($is_header != 0) {
file_put_contents($articleHandle, $current_article['xref'] . "\n", FILE_APPEND);
}
$is_header = 0; $is_header = 0;
$lines ++; $lines ++;
} }
@ -342,11 +345,14 @@ function get_articles($ns, $group)
} }
if (stripos($response, "Newsgroups: ") === 0) { if (stripos($response, "Newsgroups: ") === 0) {
$response = str_ireplace($group, $group, $response); $response = str_ireplace($group, $group, $response);
// Identify each group name for xref
$groupnames = explode("Newsgroups: ", $response);
$allgroups = preg_split("/\ |\,/", $groupnames[1]);
$ref = 0; $ref = 0;
} }
if (stripos($response, "Xref: ") === 0) { if (stripos($response, "Xref: ") === 0) {
if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) { if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) {
$response = "Xref: " . $CONFIG['pathhost'] . " " . $group . ":" . $local; $is_xref = true;
} }
$xref = $response; $xref = $response;
$ref = 0; $ref = 0;
@ -368,7 +374,9 @@ function get_articles($ns, $group)
} else { } else {
$body .= $response . "\n"; $body .= $response . "\n";
} }
if (! $is_xref) {
file_put_contents($articleHandle, $response . "\n", FILE_APPEND); file_put_contents($articleHandle, $response . "\n", FILE_APPEND);
}
// Check here for broken $ns connection before continuing // Check here for broken $ns connection before continuing
$response = fgets($ns, 1200); $response = fgets($ns, 1200);
if ($response == false) { if ($response == false) {
@ -386,7 +394,6 @@ function get_articles($ns, $group)
if ($banned != false) { if ($banned != false) {
unlink($grouppath . "/" . $local); unlink($grouppath . "/" . $local);
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Skipping: " . $CONFIG['remote_server'] . " " . $group . ":" . $article . " banned in " . $banned, FILE_APPEND); file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Skipping: " . $CONFIG['remote_server'] . " " . $group . ":" . $article . " banned in " . $banned, FILE_APPEND);
// file_put_contents($logfile, "\nFrom: ".$from[1]."\nPath: ".$msgpath[1], FILE_APPEND);
$article ++; $article ++;
} else { } else {
if ((strpos($CONFIG['nocem_groups'], $group) !== false) && ($CONFIG['enable_nocem'] == true)) { if ((strpos($CONFIG['nocem_groups'], $group) !== false) && ($CONFIG['enable_nocem'] == true)) {
@ -402,47 +409,43 @@ function get_articles($ns, $group)
copy($grouppath . "/" . $local, $bbsmail_filename); copy($grouppath . "/" . $local, $bbsmail_filename);
} }
} }
// Overview
$stmt->execute([
$group,
$local,
$mid[1],
$article_date,
$finddate[1],
$from[1],
$subject[1],
$references,
$bytes,
$lines,
$xref
]);
$references = "";
if ($CONFIG['article_database'] == '1') { if ($CONFIG['article_database'] == '1') {
$this_article = file_get_contents($grouppath . "/" . $local); $this_article = file_get_contents($grouppath . "/" . $local);
// CREATE SEARCH SNIPPET // CREATE SEARCH SNIPPET
$this_snippet = get_search_snippet($body, $content_type[1]); $this_snippet = get_search_snippet($body, $content_type[1]);
$article_stmt->execute([
$group,
$local,
$mid[1],
$article_date,
$from[1],
$subject[1],
$this_article,
$this_snippet
]);
unlink($grouppath . "/" . $local);
} else { } else {
if ($article_date > time()) if ($article_date > time()) {
$article_date = time(); $article_date = time();
}
touch($grouppath . "/" . $local, $article_date); touch($grouppath . "/" . $local, $article_date);
} }
echo "\nRetrieved: " . $group . " " . $article;
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Wrote to spool: " . $CONFIG['remote_server'] . " " . $group . ":" . $article, FILE_APPEND); $current_article['mid'] = $mid[1];
$status = "spooled"; $current_article['epochdate'] = $article_date;
$statusdate = time(); $current_article['stringdate'] = $finddate[1];
$statusreason = "imported"; $current_article['from'] = $from[1];
add_to_history($group, $local, $mid[1], $status, $statusdate, $statusreason, $statusnotes); $current_article['subject'] = $subject[1];
$current_article['references'] = $references;
$current_article['bytes'] = $bytes;
$current_article['lines'] = $lines;
$current_article['article'] = $this_article;
$current_article['snippet'] = $this_snippet;
foreach ($allgroups as $agroup) {
$agroup = trim($agroup);
if ((! testGroup($agroup)) || $agroup == '') {
continue;
}
$current_article['group'] = $agroup;
if ($group == $agroup) {
$current_article['local'] = $local;
insert_article_from_array($current_article);
} else {
$current_article['local'] = get_next_article_number($agroup);
insert_article_from_array($current_article);
}
}
$i ++; $i ++;
$article ++; $article ++;
$local ++; $local ++;
@ -451,6 +454,7 @@ function get_articles($ns, $group)
} }
} }
} }
// END GET INDIVIDUAL ARTICLE
$article --; $article --;
// $local--; // $local--;
// Update title // Update title