rocksolid-light/Rocksolid_Light/rslight/scripts/spoolnews.php

595 lines
24 KiB
PHP

<?php
/*
* spoolnews NNTP news spool creator
* Download: https://news.novabbs.com/getrslight
*
* E-Mail: retroguy@novabbs.com
* Web: https://news.novabbs.com
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
include "config.inc.php";
include ("$file_newsportal");
include $config_dir . '/gpg.conf';
if ($CONFIG['remote_server'] != '') {
$remote_groupfile = $spooldir . "/" . $config_name . "/" . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'] . ".txt";
}
if (isset($OVERRIDES['save_nocem_messages']) && $OVERRIDES['save_nocem_messages'] == true) {
$save_nocem_messages = true;
$nocem_dir = $spooldir . "/saved_nocem";
@mkdir($nocem_dir . '/done', 0755, true);
} else {
$save_nocem_messages = false;
}
$file_groups = $config_path . "groups.txt";
$local_groupfile = $spooldir . "/" . $config_name . "/local_groups.txt";
$logfile = $logdir . '/spoolnews.log';
$spamlog = $logdir . '/spam.log';
# END MAIN CONFIGURATION
@mkdir($spooldir . "/" . $config_name, 0755, 'recursive');
// Defaults
$maxarticles_per_run = 100;
$maxfirstrequest = 100;
// Overrides
if ($OVERRIDES['maxarticles_per_run'] > 0) {
$maxarticles_per_run = $OVERRIDES['maxarticles_per_run'];
}
if ($OVERRIDES['maxfirstrequest'] > 0) {
$maxfirstrequest = $OVERRIDES['maxfirstrequest'];
}
if (! isset($CONFIG['enable_nntp']) || $CONFIG['enable_nntp'] != true) {
$maxfirstrequest = $maxarticles;
$maxarticles_per_run = $maxfetch;
}
$workpath = $spooldir . "/";
$path = $workpath . "articles/";
if ($low_spool_disk_space) {
print "Low Disk Space (less than " . $min_spool_disk_space . " available)\n";
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Low Disk Space (less than " . $min_spool_disk_space . "Gb available for spool). Pausing spoolnews", FILE_APPEND);
exit();
}
$lockfile = $lockdir . '/' . $config_name . '-spoolnews.lock';
$pid = file_get_contents($lockfile);
if (posix_getsid($pid) === false || ! is_file($lockfile)) {
print "Starting Spoolnews...\n";
file_put_contents($lockfile, getmypid()); // create lockfile
} else {
print "Spoolnews currently running\n";
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Spoolnews currently running...", FILE_APPEND);
exit();
}
$sem = $spooldir . "/" . $config_name . ".reload";
if (is_file($sem)) {
unlink($remote_groupfile);
unlink($sem);
$maxfirstrequest = 200;
}
# Check for groups file, create if necessary
// only do remote server groups if necessary
if ($CONFIG['remote_server'] != '') {
create_spool_groups($file_groups, $remote_groupfile);
}
create_spool_groups($file_groups, $local_groupfile);
# Iterate through groups
$enable_rslight = 0;
# Refresh group list
$menulist = file($config_dir . "menu.conf", FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach ($menulist as $menu) {
if (($menu[0] == '#') || (trim($menu) == "")) {
continue;
}
$menuitem = explode(':', $menu);
if (($menuitem[0] == $config_name) && ($menuitem[1] == '1')) {
groups_read($server, $port, 1, true); // 'true' forces a refresh of the group list
$enable_rslight = 1;
echo "\nLoaded groups";
}
}
# Clean outgoing directory for LOCAL sections
if ($CONFIG['remote_server'] == '') {
$outgoing_dir = $spooldir . "/" . $config_name . "/outgoing/";
$files = scandir($outgoing_dir);
foreach ($files as $file) {
$file_name = $outgoing_dir . $file;
if (is_file($file_name) && (filemtime($file_name) < (time() - 3600))) {
unlink($file_name);
}
}
}
if ($CONFIG['remote_server'] != '') {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " remote_server: " . $CONFIG['remote_server'], FILE_APPEND);
$ns = nntp2_open($CONFIG['remote_server'], $CONFIG['remote_port']);
if (! $ns) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Failed to connect to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'], FILE_APPEND);
exit();
}
$grouplist = file($config_dir . '/' . $config_name . '/groups.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach ($grouplist as $findgroup) {
if ($findgroup[0] == ":") {
continue;
}
$name = preg_split("/( |\t)/", $findgroup, 2);
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Retrieving articles for: " . $name[0] . "...", FILE_APPEND);
echo "\nRetrieving articles for: " . $name[0] . "...";
get_articles($ns, $name[0]);
if ($enable_rslight == 1) {
$timer_file = $spooldir . '/tmp/' . $name[0] . '-thread-timer';
if (filemtime($timer_file) + 600 < time()) {
touch($timer_file);
if (! $ns2) {
$ns2 = nntp_open();
echo "\nOPENING $ns2: " . $ns2 . "\n";
}
if (! $ns2) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Failed to connect to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'], FILE_APPEND);
// exit();
} else {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Updating threads for: " . $name[0] . "...", FILE_APPEND);
try {
thread_load_newsserver($ns2, $name[0], 0);
} catch (Exception $exc) {
echo "\nFatal exception caught: " . $exc->getMessage();
} catch (Error $err) {
echo "\nFatal error caught: " . $err->getMessage();
}
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Threads updated for: " . $name[0], FILE_APPEND);
}
}
}
}
if ($ns2) {
nntp_close($ns2);
}
nntp_close($ns);
}
unlink($lockfile);
echo "\nSpoolnews Done\n";
function get_articles($ns, $group)
{
global $enable_rslight, $rslight_gpg, $spooldir, $nocem_dir, $save_nocem_messages, $CONFIG, $OVERRIDES, $user_ban_file, $maxarticles_per_run, $maxfirstrequest, $workpath, $path, $remote_groupfile, $local_groupfile, $local, $logdir, $config_name, $spamlog, $logfile;
if ($ns == false) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Lost connection to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'], FILE_APPEND);
exit();
}
$grouppath = $path . preg_replace('/\./', '/', $group);
// $banned_names = file($user_ban_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$msgid_filter = get_config_value('header_filters.conf', 'Message-ID');
$subject_filter = get_config_value('header_filters.conf', 'Subject');
$from_filter = get_config_value('header_filters.conf', 'From');
$path_filter = get_config_value('header_filters.conf', 'Path');
$nocem_check = "@@NCM";
$bbsmail_check = "@@RSL";
# Check if group exists. Open it if it does
fputs($ns, "group " . $group . "\r\n");
$response = line_read($ns);
if (strcmp(substr($response, 0, 3), "211") != 0) {
echo "\n" . $response;
return (1);
}
# Get config
$grouplist = file($remote_groupfile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
foreach ($grouplist as $findgroup) {
$name = explode(':', $findgroup);
if (strcmp($name[0], $group) == 0) {
if (is_numeric(trim($name[1]))) {
$article = $name[1] + 1;
} else {
$article = 1;
}
break;
}
}
if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) {
// Get next available article number for group
$local = get_next_article_number($group);
}
# Split group response line to get last article number
$detail = explode(" ", $response);
if (! isset($article)) {
$article = $detail[2];
}
if ($article < $detail[3] - $maxfirstrequest) {
$article = $detail[3] - $maxfirstrequest;
}
if ($article < $detail[2]) {
$article = $detail[2];
}
// Get overview from server
$server_overview = array();
$re = false;
if (($detail[3] - $article) > $maxarticles_per_run) {
$getlast = $article + $maxarticles_per_run;
} else {
$getlast = $detail[3];
}
fputs($ns, "xover " . $article . "-" . $getlast . "\r\n");
$response = line_read($ns); // and once more
if ((substr($response, 0, 3) != "224")) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Cannot get overview from " . $CONFIG['remote_server'] . " for " . $group, FILE_APPEND);
return false;
}
while (trim($response = line_read($ns)) !== '.') {
$ov = preg_split("/\t/", $response);
$overview_msgid[$ov[0]] = $ov[4];
}
# Pull articles and save them in our spool
if (! is_dir($grouppath)) {
mkdir($grouppath, 0755, 'recursive');
}
$i = 0;
// GET INDIVIDUAL ARTICLE
while ($article <= $detail[3]) {
if (! is_numeric($article)) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " DEBUG This should show server group:article number: " . $CONFIG['remote_server'] . " " . $group . ":" . $article, FILE_APPEND);
break;
}
// Create array for article, then send to insert_article_from_array()
if (isset($current_article)) {
unset($current_article);
$current_article = array();
}
if ($CONFIG['enable_nntp'] != true) {
$local = $article;
}
if (check_duplicate_msgid($overview_msgid[$article], $group)) {
echo "\n(spoolnews)Duplicate Message-ID for: " . $group . ":" . $overview_msgid[$article];
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Duplicate Message-ID for: " . $group . ":" . $overview_msgid[$article], FILE_APPEND);
$article ++;
continue;
}
fputs($ns, "article " . $article . "\r\n");
$response = line_read($ns);
if (strcmp(substr($response, 0, 3), "220") != 0) {
echo "\n" . $response;
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " " . $group . " " . $response, FILE_APPEND);
$article ++;
continue;
}
$articleHandle = $grouppath . "/" . $local;
unlink($articleHandle);
unset($references);
$response = line_read($ns);
$lines = 0;
$bytes = 0;
$ref = 0;
$sub = 0;
$banned = false;
$integrity = false;
$is_header = 1;
$body = "";
$content_transfer_encoding = null;
while (strcmp($response, ".") != 0) {
$is_xref = false;
$bytes = $bytes + mb_strlen($response, '8bit');
if (trim($response) == "" && $lines > 0) {
if ($is_header == 1) {
file_put_contents($articleHandle, $current_article['xref'] . "\n", FILE_APPEND);
}
$is_header = 0;
}
if ($is_header == 1) {
$response = str_replace("\t", " ", $response);
if (strpos($response, ': ') !== false) {
$ref = 0;
$sub = 0;
}
// Find article date
if (stripos($response, "Date: ") === 0) {
$finddate = explode(': ', $response, 2);
$article_date = strtotime($finddate[1]);
}
// Get overview data
if (stripos($response, "Message-ID: ") === 0) {
$mid = explode(': ', $response, 2);
if (preg_match($msgid_filter, $mid[1])) {
$banned = "msgid_filter";
}
}
if (stripos($response, "From: ") === 0) {
$from = explode(': ', $response, 2);
if (preg_match($from_filter, $from[1])) {
$banned = "from_filter";
}
}
if (stripos($response, "Path: ") === 0) {
$msgpath = explode(': ', $response, 2);
if (preg_match($path_filter, $msgpath[1])) {
$banned = "path_filter";
}
}
if (stripos($response, "Subject: ") === 0) {
$this_subject = explode('Subject: ', $response, 2);
$subject = $this_subject[1];
$sub = 1;
if (preg_match($subject_filter, $subject)) {
$banned = "subject_filter";
}
}
// Transfer encoding
if (stripos($response, "Content-Transfer-Encoding: ") === 0) {
$enco = explode(': ', $response, 2);
$content_transfer_encoding = $enco[1];
}
if (stripos($response, "Newsgroups: ") === 0) {
$response = str_ireplace($group, $group, $response);
// Identify each group name for xref
$groupnames = explode("Newsgroups: ", $response);
$allgroups = preg_split("/\ |\,/", $groupnames[1]);
// Create Xref: header
$current_article['xref'] = "Xref: " . $CONFIG['pathhost'];
foreach ($allgroups as $agroup) {
$agroup = trim($agroup);
if ((! testGroup($agroup)) || $agroup == '') {
continue;
}
if ($group == $agroup) {
$artnum = $local;
} else {
$artnum = get_next_article_number($agroup);
}
if ($artnum > 0) {
$current_article['xref'] .= ' ' . $agroup . ':' . $artnum;
}
}
}
if (stripos($response, "Xref: ") === 0) {
if (isset($CONFIG['enable_nntp']) && $CONFIG['enable_nntp'] == true) {
$is_xref = true;
}
$xref = $response;
}
if (stripos($response, "Content-Type: ") === 0) {
preg_match('/.*charset=.*/', $response, $te);
$content_type = explode("Content-Type: text/plain; charset=", $te[0]);
}
if (stripos($response, "References: ") === 0) {
$this_references = explode('References: ', $response);
$references = $this_references[1];
$ref = 1;
}
if (preg_match('/^\s/', $response) && $ref == 1) {
$references = $references . $response;
}
if (preg_match('/^\s/', $response) && $sub == 1) {
$subject = $subject . $response;
}
} else {
$body .= $response . "\n";
}
if (! $is_xref) {
file_put_contents($articleHandle, $response . "\n", FILE_APPEND);
}
// Check here for broken $ns connection before continuing
$response = fgets($ns, 1200);
if ($response == false) {
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Lost connection to " . $CONFIG['remote_server'] . ":" . $CONFIG['remote_port'] . " retrieving article " . $article, FILE_APPEND);
unlink($articleHandle);
break;
// continue;
}
$response = str_replace("\n", "", str_replace("\r", "", $response));
$lines ++;
}
file_put_contents($articleHandle, $response . "\n", FILE_APPEND);
$lines = $lines - 1;
$bytes = $bytes + ($lines * 2);
// Don't spool article if $banned or fails integrity test
$integrity = check_article_integrity(file($articleHandle));
if (($banned !== false) || ($integrity !== false)) {
unlink($articleHandle);
if($integrity) {
file_put_contents($logfile, "\n" . format_log_date() . $integrity, FILE_APPEND);
} elseif ($banned) {
file_put_contents($spamlog, "\n" . format_log_date() . " " . $banned . " :\tSPAM\t" . $mid[1] . "\t" . $groupnames[1] . "\t" . $from[1], FILE_APPEND);
}
$article ++;
} else {
if ((strpos($CONFIG['nocem_groups'], $group) !== false) && ($CONFIG['enable_nocem'] == true)) {
if (strpos($subject, $nocem_check) !== false) {
$is_from = address_decode($from[1], 'nowhere');
$nocem_file = tempnam($spooldir . "/nocem", $is_from[0]['mailbox'] . "@" . $is_from[0]['host'] . "[" . date("Y.m.d.H.i.s") . "]");
copy($articleHandle, $nocem_file);
chmod($nocem_file, 0644);
if ($save_nocem_messages == true) {
$saved_nocem_file = tempnam($nocem_dir, $is_from[0]['mailbox'] . "@" . $is_from[0]['host'] . "[" . date("Y.m.d.H.i.s") . "]-");
copy($articleHandle, $saved_nocem_file);
}
}
}
if ((strpos($rslight_gpg['nntp_group'], $group) !== false) && ($rslight_gpg['enable'] == '1')) {
if (strpos($subject, $bbsmail_check) !== false) {
$bbsmail_file = preg_replace('/@@RSL /', '', $subject);
$bbsmail_filename = $spooldir . "/bbsmail/in/bbsmail-" . $bbsmail_file;
copy($articleHandle, $bbsmail_filename);
}
}
$this_article = file_get_contents($articleHandle);
if ($CONFIG['article_database'] == '1') {
unlink($articleHandle);
// CREATE SEARCH SNIPPET
$this_snippet = get_search_snippet($body, $content_type[1], $content_transfer_encoding);
} else {
if ($article_date > time()) {
$article_date = time();
}
touch($articleHandle, $article_date);
}
$current_article['mid'] = $mid[1];
$current_article['epochdate'] = $article_date;
$current_article['stringdate'] = $finddate[1];
$current_article['from'] = $from[1];
$current_article['subject'] = $subject;
$current_article['references'] = $references;
$current_article['bytes'] = $bytes;
$current_article['lines'] = $lines;
$current_article['article'] = $this_article;
$current_article['snippet'] = $this_snippet;
// Check Spam
$res = 0;
if (isset($CONFIG['spamassassin']) && ($CONFIG['spamassassin'] == true) && ($OVERRIDES['disable_spamassassin_spooling'] !== true)) {
$spam_result_array = check_spam($subject, $from[1], $groupnames[1], $references, $this_article, $mid[1]);
$res = $spam_result_array['res'];
$spamresult = $spam_result_array['spamresult'];
$spamcheckerversion = $spam_result_array['spamcheckerversion'];
$spamlevel = $spam_result_array['spamlevel'];
}
if ($res === 1) {
unlink($articleHandle);
file_put_contents($logfile, "\n" . format_log_date() . " " . $config_name . " Skipping: " . $CONFIG['remote_server'] . " " . $group . ":" . $article . " Exceeds Spam Score", FILE_APPEND);
// $orig_newsgroups = $newsgroups;
// $newsgroups = $CONFIG['spamgroup'];
// $group = $newsgroups;
$i --;
$local --;
} else {
$pass = false;
foreach ($allgroups as $agroup) {
$agroup = trim($agroup);
if ((! testGroup($agroup)) || $agroup == '') {
continue;
}
$current_article['group'] = $agroup;
if ($group == $agroup) {
$current_article['local'] = $local;
} else {
$current_article['local'] = get_next_article_number($agroup);
}
$tmp = insert_article_from_array($current_article, false);
if ($tmp[0] != "4") {
$pass = true;
}
}
if (! $pass) {
$i --;
}
}
$i ++;
$article ++;
$local ++;
if ($i > $maxarticles_per_run) {
break;
}
}
}
// END GET INDIVIDUAL ARTICLE
$article --;
// $local--;
// Update title
if (! is_file($workpath . $group . "-title")) {
fputs($ns, "XGTITLE " . $group . "\r\n");
$response = line_read($ns);
if (strcmp(substr($response, 0, 3), "282") == 0) {
$titlefile = $workpath . $group . "-title";
$response = line_read($ns);
while (strcmp($response, ".") != 0) {
file_put_contents($titlefile, $response);
$response = line_read($ns);
}
}
}
# Save config
$grouplist = file($remote_groupfile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$saveconfig = fopen($remote_groupfile, 'w+');
foreach ($grouplist as $savegroup) {
$name = explode(':', $savegroup);
if (strcmp($name[0], $group) == 0) {
fputs($saveconfig, $group . ":" . $article . "\n");
} else {
fputs($saveconfig, $savegroup . "\n");
}
}
fclose($saveconfig);
$grouplist = file($local_groupfile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$saveconfig = fopen($local_groupfile, 'w+');
foreach ($grouplist as $savegroup) {
$name = explode(':', $savegroup);
if (strcmp($name[0], $group) == 0) {
fputs($saveconfig, $group . ":" . $local . "\n");
} else {
fputs($saveconfig, $savegroup . "\n");
}
}
fclose($saveconfig);
if ($CONFIG['article_database'] == '1') {
$article_dbh = null;
}
$dbh = null;
}
function create_spool_groups($in_groups, $out_groups)
{
global $spooldir;
$grouplist = file($in_groups, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$temp_file = tempnam($spooldir . "/tmp/", 'groupfile-');
foreach ($grouplist as $group) {
if ($group[0] == ":") {
continue;
}
$thisgroup = preg_split("/( |\t)/", $group, 2);
if ($val = get_config_file_value($out_groups, $thisgroup[0])) {
file_put_contents($temp_file, $thisgroup[0] . ":" . $val . "\n", FILE_APPEND);
} else {
file_put_contents($temp_file, $thisgroup[0] . "\n", FILE_APPEND);
}
}
rename($temp_file, $out_groups);
return;
}
function get_article_list($thisgroup)
{
global $spooldir;
$database = $spooldir . "/articles-overview.db3";
$table = 'overview';
$dbh = overview_db_open($database, $table);
$stmt = $dbh->prepare("SELECT * FROM $table WHERE newsgroup=:thisgroup ORDER BY number");
$stmt->execute([
'thisgroup' => $thisgroup
]);
$ok_article = array();
while ($found = $stmt->fetch()) {
$ok_article[] = $found['number'];
}
$dbh = null;
return (array_unique($ok_article));
}
?>